Compare commits

...
Author SHA1 Message Date
bsodfather 4832878468 refactor(mcp): migrate compatible logging paths to rmcp SDK
CI / UI Checks (pull_request) Successful in 9s
CI / Rust Checks (pull_request) Successful in 27m19s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
2026-11-11 22:51:42 +00:00
bsodfather 5bf4e3e9bf Merge pull request 'fix(deps): rustix 1.1.6 for release security gate' (#125) from fix/rustix-rustsec-2026-0486 into main
CI / UI Checks (push) Successful in 13s
CI / Rust Checks (push) Successful in 34m16s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 1s
CI / Deploy (push) Skipped
Community Release / release (push) Successful in 52m54s
2026-11-11 12:10:13 +00:00
bsodfather 1ac5b10c67 fix(deps): update rustix for RUSTSEC-2026-0486
CI / UI Checks (pull_request) Successful in 7s
CI / Rust Checks (pull_request) Successful in 27m46s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
2026-11-11 11:41:23 +00:00
bsodfather 07a9ef0a4a Merge pull request 'refactor(mcp): completion request/result via rmcp SDK' (#124) from delivery/rmcp107-completion into main
CI / UI Checks (push) Successful in 12s
CI / Rust Checks (push) Successful in 31m41s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 1s
CI / Deploy (push) Skipped
Community Release / release (push) Failing after 12s
2026-11-11 10:52:36 +00:00
bsodfather 4f83d8d24c refactor(mcp): route completion requests and results through rmcp SDK
CI / UI Checks (pull_request) Successful in 12s
CI / Rust Checks (pull_request) Successful in 36m53s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
2026-11-11 10:11:35 +00:00
bsodfather 2a57800f63 Merge pull request 'fix(mcp): rmcp 3.9.0 — RUSTSEC-2026-0462' (#123) from fix/rmcp-security-0462 into main
CI / UI Checks (push) Successful in 9s
CI / Rust Checks (push) Successful in 25m46s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 3s
CI / Deploy (push) Skipped
Community Release / release (push) Successful in 48m48s
2026-11-10 21:51:10 +00:00
bsodfather e359077f6e fix(mcp): update rmcp to 3.9.0 for RUSTSEC-2026-0462
CI / UI Checks (pull_request) Successful in 11s
CI / Rust Checks (pull_request) Successful in 32m46s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
2026-11-10 21:17:38 +00:00
bsodfather c0117ff6b8 Merge pull request 'refactor(mcp): migrate Prompts list/get through rmcp SDK' (#122) from delivery/rmcp107-prompts into main
CI / UI Checks (push) Successful in 13s
CI / Rust Checks (push) Successful in 31m52s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 5s
CI / Deploy (push) Skipped
Community Release / release (push) Failing after 12s
2026-11-10 19:51:33 +00:00
bsodfather 24bba2cf33 refactor(mcp): migrate Prompts requests and results through rmcp SDK
CI / UI Checks (pull_request) Successful in 11s
CI / Rust Checks (pull_request) Successful in 30m13s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Deploy (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
2026-11-10 19:17:54 +00:00
bsodfather f56553c4d6 Merge pull request 'refactor(mcp): Resources request/result runtime through rmcp SDK' (#121) from delivery/rmcp107-resources into main
CI / UI Checks (push) Successful in 12s
CI / Rust Checks (push) Successful in 25m54s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 2s
CI / Deploy (push) Skipped
Community Release / release (push) Successful in 56m48s
2026-11-09 05:16:06 +00:00
bsodfather 027e2cb911 refactor(mcp): route Resources requests and results through rmcp SDK
CI / UI Checks (pull_request) Successful in 12s
CI / Rust Checks (pull_request) Successful in 30m41s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
2026-11-09 04:42:43 +00:00
bsodfather adabb2065a Merge pull request 'Serialize MCP Tools responses through rmcp SDK models (partial #107)' (#120) from delivery/rmcp107-tools-responses into main
CI / UI Checks (push) Successful in 8s
CI / Rust Checks (push) Successful in 33m25s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 1s
CI / Deploy (push) Skipped
Community Release / release (push) Successful in 51m19s
2026-11-08 23:19:11 +00:00
bsodfather bf773d61b8 feat(mcp): serialize Tools responses through rmcp SDK models
CI / UI Checks (pull_request) Successful in 10s
CI / Rust Checks (pull_request) Successful in 24m38s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
2026-11-08 22:52:00 +00:00
bsodfather 74a98d9362 Merge pull request 'Route existing MCP Tools requests through typed rmcp runtime seam (partial #107)' (#119) from feat/rmcp107-tools-runtime into main
CI / Rust Checks (push) Successful in 24m5s
CI / Rust Feature Validation (push) Skipped
CI / UI Checks (push) Successful in 7s
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 1s
CI / Deploy (push) Skipped
Community Release / release (push) Failing after 45m55s
2026-11-08 15:30:17 +00:00
bsodfather 7f18bce679 feat(mcp): route Tools requests through typed rmcp runtime seam
CI / Rust Checks (pull_request) Successful in 24m17s
CI / Rust Feature Validation (pull_request) Skipped
CI / UI Checks (pull_request) Successful in 7s
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
2026-11-08 15:00:22 +00:00
bsodfather 4ec9d9201b Merge pull request 'Use verified Trivy runner cache and avoid Cargo builds for release-only fixes' (#118) from fix/trivy-runner-cache-20261108 into main
CI / UI Checks (push) Successful in 14s
CI / Rust Checks (push) Failing after 1h39m35s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Failing after 1s
CI / Deploy (push) Skipped
2026-11-08 12:26:19 +00:00
bsodfather cf2909273b perf(ci): scope release tooling changes without unrelated Cargo builds
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 24m39s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
2026-11-08 14:47:35 +03:00
bsodfather 4a7b230f35 fix(ci): load checksum-verified Trivy from runner tool cache 2026-11-08 14:21:16 +03:00
bsodfather cfe11a0a04 Merge pull request 'fix(ci): download pinned Trivy asset through GitHub API on HTTP failure' (#117) from fix/trivy-asset-api-20261108 into main
CI / UI Checks (push) Successful in 14s
CI / Rust Checks (push) Failing after 17m52s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Failing after 2s
CI / Deploy (push) Skipped
2026-11-08 10:59:43 +00:00
bsodfather 35a21f8ab5 fix(ci): fall back to pinned GitHub asset API for Trivy
CI / UI Checks (pull_request) Successful in 11s
CI / Rust Checks (pull_request) Successful in 25m14s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
2026-11-08 13:02:56 +03:00
bsodfather 2d0deb8963 Merge pull request 'fix(ci): download Trivy database from reachable official registry' (#116) from fix/release-db-source-20261107 into main
CI / UI Checks (push) Successful in 14s
CI / Rust Checks (push) Successful in 25m23s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 1s
CI / Deploy (push) Skipped
2026-11-08 09:40:01 +00:00
bsodfather f0a79362eb fix(ci): prefer reachable official Trivy database source
CI / UI Checks (pull_request) Successful in 11s
CI / Rust Checks (pull_request) Successful in 30m33s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
2026-11-07 18:24:08 +03:00
bsodfather 27f86d4916 Merge pull request 'fix(ci): continue interrupted releases without replacing assets' (#115) from fix/release-resume-20261107 into main
CI / UI Checks (push) Successful in 10s
CI / Rust Checks (push) Successful in 33m36s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 1s
CI / Deploy (push) Skipped
fix(ci): safely resume incomplete immutable releases
2026-11-07 14:49:41 +00:00
bsodfather f728d31dbf fix(ci): safely resume incomplete immutable releases
CI / UI Checks (pull_request) Successful in 9s
CI / Rust Checks (pull_request) Successful in 24m48s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 3s
CI / Deploy (pull_request) Skipped
2026-11-07 17:24:26 +03:00
bsodfather 29ee01b6ef Merge pull request 'Prototype isolated rmcp Tools adapter (#106)' (#114) from feat/rmcp106-tools-adapter into main
CI / UI Checks (push) Successful in 9s
CI / Rust Checks (push) Successful in 24m5s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 1s
CI / Deploy (push) Skipped
Community Release / release (push) Failing after 35m1s
2026-11-07 08:57:25 +00:00
bsodfather bcf8b43516 fix(ci): reuse installed pinned Rust toolchain
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 24m46s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 2s
CI / Deploy (pull_request) Skipped
2026-11-07 11:29:53 +03:00
bsodfather 1c256e6c75 Add isolated rmcp Tools adapter prototype
CI / Rust Checks (pull_request) Failing after 40s
CI / Rust Feature Validation (pull_request) Skipped
CI / UI Checks (pull_request) Successful in 12s
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Failing after 2s
CI / Deploy (pull_request) Skipped
2026-11-06 07:56:21 +00:00
bsodfather e4953abcd2 Merge pull request 'Story 4.5 — продолжение ожидающей Task после ввода' (#113) from feat/story-4.5-input-continuation into main
CI / UI Checks (push) Successful in 14s
CI / Rust Checks (push) Successful in 28m27s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 2s
CI / Deploy (push) Skipped
Community Release / release (push) Successful in 48m41s
2026-10-30 16:47:18 +00:00
bsodfather 90e5357a08 feat(epic4): продолжение ожидающей Task после ввода (Story 4.5)
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 21m42s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Durable worker приостанавливает Task как input_required до dispatch, если
закреплённая схема требует отсутствующее поле; authorized schema-validated
update атомарно и с аудитом возобновляет тот же Task ровно один раз.
Replay/conflict/TTL безопасны, ожидание удерживает admission permit.
Additive schema v30: task_input_requests, fingerprint/version, TTL-индекс.
2026-10-30 16:24:12 +00:00
bsodfather d80eda7329 Merge pull request 'Story 4.4 — безопасные повторные попытки и отмена durable Task' (#112) from delivery/epic4-story-4.4 into main
CI / UI Checks (push) Successful in 11s
CI / Rust Checks (push) Successful in 27m33s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 1s
CI / Deploy (push) Skipped
Community Release / release (push) Successful in 44m29s
Merge pull request 'Story 4.4 — безопасные повторные попытки и отмена durable Task' (#112) from delivery/epic4-story-4.4 into main
2026-10-30 12:46:29 +00:00
bsodfather a3057de51e feat(epic4): безопасные повторы и отмена durable Task (Story 4.4)
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 20m46s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Durable per-outbound attempt до отправки запроса, bounded retry/read,
server-owned verified writes, честный cancel/unknown, deadline/fencing,
additive schema v29. Staged renames task.rs/tasks.rs разложены на модули.

Includes disclosed cosmetic parity exception: единственная правка вне
принятой дельты — удалена лишняя пустая строка в конце
scripts/ci-story-checks.sh (EOF blank line); story-код не менялся.
2026-10-30 12:21:57 +00:00
bsodfather 8c02380762 fix(ui): временно ограничить npm build-стадию UI TLS 1.2 на CI-хосте
CI / UI Checks (push) Successful in 9s
CI / Rust Checks (push) Successful in 25m23s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 3s
CI / Deploy (push) Skipped
Merge pull request #111 'fix(ui): временно ограничить npm build-стадию UI TLS 1.2 на CI-хосте' from delivery/ui-npm-tls12-cap
2026-10-30 10:51:38 +00:00
hermes_dev 3b5fcc51cc fix(ui): pin npm build stage to TLS 1.2 on the CI host uplink
CI / UI Checks (pull_request) Successful in 9s
CI / Rust Checks (pull_request) Successful in 20m41s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
The CI host uplink (192.168.1.108) drops TLS 1.3 ClientHello carrying the
hybrid X25519MLKEM768 key share, so registry.npmjs.org is unreachable from
the seeded node:22-alpine build stage while the same host answers over TLS
1.2. RUN npm ci hung until the build was cancelled.

Cap only the UI build stage at TLS 1.2 via NODE_OPTIONS=--tls-max-v1.2.
Certificates, SRI and the lockfile are untouched; no mirror, fallback or
cache-substituted packages. Remove the line once TLS 1.3 works from the CI
host again.
2026-10-30 09:56:19 +00:00
bsodfather 433987413e Merge pull request 'Story 4.3 — исполнять и восстанавливать durable Task через PG lease/fence' (#110) from delivery/epic4-story-4.3 into main
CI / UI Checks (push) Successful in 8s
CI / Rust Checks (push) Successful in 26m20s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 2s
CI / Deploy (push) Skipped
Community Release / release (push) Failing after 31m2s
2026-10-29 23:08:04 +00:00
bsodfather 5488eb73ec chore(migrations): зафиксировать каноническую последовательность v26–v28
CI / Rust Checks (pull_request) Successful in 22m6s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
CI / UI Checks (pull_request) Successful in 12s
Обязательный CI-гейт «Check canonical migration contract» сравнивает
`crank-migrate plan` с docs/schemas/migration-sequence.json. Принятая дельта
Story 4.3 добавила миграции v26–v28, но снимок контракта не был регенерирован,
поэтому exact-head CI падал на точном PR head с contract_drift (plan.compare).

Файл получен машинной регенерацией из того же дерева:
`crank-migrate plan > docs/schemas/migration-sequence.json`; проверка
`crank-migrate plan --check` возвращает {"status":"contract_current"}.
Содержимое меняется только добавлением трёх записей 26/27/28; существующие
записи 1–25 и их checksum не изменены, поведение продукта не меняется.
2026-10-29 22:43:05 +00:00
bsodfather f9f8c625d1 feat(epic4): исполнять durable Task через PG lease/fence и восстанавливать после сбоев (Story 4.3)
CI / UI Checks (pull_request) Successful in 15s
CI / Rust Checks (pull_request) Failing after 4m58s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Failing after 1s
CI / Deploy (pull_request) Skipped
mcp-server запускает сохранённые фоновые задачи: захват по PostgreSQL lease с
fence-токеном, зашифрованный input и повторная проверка доступа до сетевого
вызова, checkpoint с AttemptID до dispatch и восстановление только по PG.
Устаревший worker не может зафиксировать результат.

Добавлены миграции v26–v28 (V25 не меняется), guard их отсутствия в старой
схеме, метрики/трассы попыток (attempt_trace) и остановка до отправки запроса:
стоп в окне после checkpoint и до permit завершает задачу как
failed(shutdown_before_dispatch) без сетевых вызовов, соседние queued-задачи
остаются доступными. Retry-попытки получают отдельные дочерние spans с durable
current_attempt_id. Shared admission/fairness из Story 4.1 сохранены.

Проверки: durable_tasks 13/13, stages 11/11 (retry 5), runtime lib 47,
bin shutdown 1, registry tasks 16, admission 11, migrations 58 (независимая QA
и исходный runtime-прогон), точный exact-head CI остаётся обязательным.
2026-10-29 22:26:52 +00:00
112 changed files with 19836 additions and 1108 deletions

No files matched your search

+34 -9
View File
@@ -40,6 +40,15 @@ jobs:
with:
fetch-depth: 0
- name: Detect release-tooling-only changes
id: release_tooling
env:
GITEA_EVENT_NAME: ${{ gitea.event_name }}
GITEA_EVENT_BEFORE: ${{ gitea.event.before }}
GITEA_PR_BASE_SHA: ${{ gitea.event.pull_request.base.sha }}
FORCE_FULL: ${{ inputs.validation_level == 'feature' || contains(gitea.event.pull_request.labels.*.name, 'feature-ready') }}
run: python3 scripts/ci-release-tooling-scope.py
- name: Install Rust toolchain
run: |
set -eu
@@ -48,9 +57,13 @@ jobs:
echo "Unable to read Rust toolchain channel from rust-toolchain.toml" >&2
exit 1
fi
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
rustup default "$toolchain"
host="$(rustc -vV | sed -n 's/^host: //p')"
if ! rustup run "$toolchain" rustc --version >/dev/null 2>&1 || \
! rustup run "$toolchain" cargo --version >/dev/null 2>&1 || \
! rustup run "$toolchain" rustfmt --version >/dev/null 2>&1 || \
! rustup run "$toolchain" cargo-clippy --version >/dev/null 2>&1; then
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
fi
host="$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')"
toolchain_dir="${RUSTUP_HOME:-$HOME/.rustup}/toolchains/${toolchain}-${host}"
toolchain_bin="$toolchain_dir/bin"
if [ ! -x "$toolchain_bin/rustc" ] || [ ! -x "$toolchain_bin/cargo" ]; then
@@ -74,15 +87,18 @@ jobs:
docker info
- name: Check typed runtime configuration contract
if: steps.release_tooling.outputs.ci_only != 'true'
run: |
cargo run -p crank-config --bin crank-config-contract -- --check
python3 scripts/check-runtime-config.py --root .
python3 scripts/check-config-boundaries.py --root .
- name: Check canonical migration contract
if: steps.release_tooling.outputs.ci_only != 'true'
run: cargo run -p admin-api --bin crank-migrate -- plan --check
- name: Check typed metrics contract
if: steps.release_tooling.outputs.ci_only != 'true'
run: |
cargo run -p crank-metrics --bin crank-metrics-contract -- --check
python3 scripts/check-metrics-boundaries.py --root .
@@ -138,6 +154,7 @@ jobs:
python3 scripts/check-openapi-matrix-compatibility.py --base "$base_ref"
- name: Run Crank story ATDD
if: steps.release_tooling.outputs.ci_only != 'true'
run: scripts/ci-story-checks.sh
ui:
name: UI Checks
@@ -187,9 +204,13 @@ jobs:
echo "Unable to read Rust toolchain channel from rust-toolchain.toml" >&2
exit 1
fi
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
rustup default "$toolchain"
host="$(rustc -vV | sed -n 's/^host: //p')"
if ! rustup run "$toolchain" rustc --version >/dev/null 2>&1 || \
! rustup run "$toolchain" cargo --version >/dev/null 2>&1 || \
! rustup run "$toolchain" rustfmt --version >/dev/null 2>&1 || \
! rustup run "$toolchain" cargo-clippy --version >/dev/null 2>&1; then
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
fi
host="$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')"
toolchain_dir="${RUSTUP_HOME:-$HOME/.rustup}/toolchains/${toolchain}-${host}"
toolchain_bin="$toolchain_dir/bin"
if [ ! -x "$toolchain_bin/rustc" ] || [ ! -x "$toolchain_bin/cargo" ]; then
@@ -266,9 +287,13 @@ jobs:
echo "Unable to read Rust toolchain channel from rust-toolchain.toml" >&2
exit 1
fi
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
rustup default "$toolchain"
host="$(rustc -vV | sed -n 's/^host: //p')"
if ! rustup run "$toolchain" rustc --version >/dev/null 2>&1 || \
! rustup run "$toolchain" cargo --version >/dev/null 2>&1 || \
! rustup run "$toolchain" rustfmt --version >/dev/null 2>&1 || \
! rustup run "$toolchain" cargo-clippy --version >/dev/null 2>&1; then
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
fi
host="$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')"
toolchain_dir="${RUSTUP_HOME:-$HOME/.rustup}/toolchains/${toolchain}-${host}"
toolchain_bin="$toolchain_dir/bin"
if [ ! -x "$toolchain_bin/rustc" ] || [ ! -x "$toolchain_bin/cargo" ]; then
+74 -8
View File
@@ -12,10 +12,20 @@ on:
options: [story, feature]
default: story
required: true
resume_tag:
description: Existing draft release tag to resume with image-only steps
type: string
default: ""
required: false
resume_source_sha:
description: Exact accepted source commit SHA for the resume tag
type: string
default: ""
required: false
env:
REGISTRY: git.itexp.me
IMAGE_TAG: ${{ gitea.ref_name }}
IMAGE_TAG: ${{ inputs.resume_tag || gitea.ref_name }}
ADMIN_API_IMAGE: git.itexp.me/bsodfather/crank-community-admin-api
MCP_SERVER_IMAGE: git.itexp.me/bsodfather/crank-community-mcp-server
UI_IMAGE: git.itexp.me/bsodfather/crank-community-ui
@@ -31,12 +41,47 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
fetch-depth: 0
- name: Verify existing release before resume
id: resume
if: inputs.resume_tag != ''
env:
GITEA_SERVER_URL: https://git.itexp.me
GITEA_REPOSITORY: ${{ gitea.repository }}
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
RELEASE_TAG: ${{ inputs.resume_tag }}
RESUME_SOURCE_SHA: ${{ inputs.resume_source_sha }}
run: python3 scripts/ci-release-resume.py
- name: Preserve updated image scanner for resume
id: pin_resume_scanner
if: inputs.resume_tag != ''
run: |
set -eu
scanner_dir="$(mktemp -d "${RUNNER_TEMP:?RUNNER_TEMP is unavailable}/crank-release-scanner.XXXXXX")"
cp scripts/scan-images.sh "$scanner_dir/scan-images.sh"
cp scripts/check-image-scan.py "$scanner_dir/check-image-scan.py"
chmod +x "$scanner_dir/scan-images.sh"
printf 'path=%s\n' "$scanner_dir/scan-images.sh" >> "$GITHUB_OUTPUT"
- name: Checkout validated release source
if: inputs.resume_tag != ''
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
ref: ${{ steps.resume.outputs.release_tag }}
fetch-depth: 0
- name: Assert exact release source checkout
if: inputs.resume_tag != ''
env:
EXPECTED_SOURCE_SHA: ${{ steps.resume.outputs.source_sha }}
run: test "$(git rev-parse HEAD)" = "$EXPECTED_SOURCE_SHA"
- name: Clear optional release evidence
if: inputs.resume_tag == ''
run: rm -f .tmp/openapi-playwright.json .tmp/openapi-ui-evidence.json
- name: Use preinstalled Rust toolchain
if: inputs.resume_tag == ''
run: |
set -eu
toolchain_dir="${RUSTUP_HOME:-$HOME/.rustup}/toolchains/1.96.1-x86_64-unknown-linux-gnu"
@@ -54,6 +99,7 @@ jobs:
"$toolchain_bin/cargo-clippy" --version
- name: Verify runner toolchain
if: inputs.resume_tag == ''
run: |
python3 --version
rustc --version
@@ -65,19 +111,23 @@ jobs:
bao version
- name: Install dependency policy tool
if: inputs.resume_tag == ''
run: cargo install cargo-deny --version 0.20.2 --locked
- name: Check formatting
if: inputs.resume_tag == ''
run: cargo fmt --all --check
- name: Audit Rust dependencies
if: inputs.resume_tag == ''
run: cargo deny --locked check advisories bans licenses sources
- name: Run tooling unit tests
if: inputs.validation_level == 'feature'
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
run: python3 -m unittest discover -s tests/unit
- name: Check typed runtime configuration contract
if: inputs.resume_tag == ''
run: |
cargo run -p crank-config --bin crank-config-contract -- --check
python3 scripts/check-runtime-config.py --root .
@@ -85,14 +135,17 @@ jobs:
scripts/check-rust-boundaries.sh
- name: Check canonical migration contract
if: inputs.resume_tag == ''
run: cargo run -p admin-api --bin crank-migrate -- plan --check
- name: Check typed metrics contract
if: inputs.resume_tag == ''
run: |
cargo run -p crank-metrics --bin crank-metrics-contract -- --check
python3 scripts/check-metrics-boundaries.py --root .
- name: Check Capability Inventory
if: inputs.resume_tag == ''
run: |
required_args=""
for number in $(seq 1 54); do
@@ -105,6 +158,7 @@ jobs:
$required_args
- name: Check Capability Baseline
if: inputs.resume_tag == ''
run: |
python3 scripts/validate-capability-baseline.py \
--root . \
@@ -112,40 +166,46 @@ jobs:
--schema docs/schemas/capability-baseline.schema.json
- name: Check Community scope
if: inputs.resume_tag == ''
run: scripts/check-community-scope.sh
- name: Run Crank story ATDD
if: inputs.resume_tag == ''
run: scripts/ci-story-checks.sh
- name: Run release quality gates
if: inputs.validation_level == 'feature'
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
run: |
cargo clippy --workspace --all-targets --all-features -- -D warnings
cargo test --workspace --all-targets -- --test-threads=1
- name: Build release binaries
if: inputs.resume_tag == ''
run: cargo build --release -p admin-api -p mcp-server
- name: Install UI dependencies
if: inputs.resume_tag == ''
working-directory: apps/ui
run: npm ci
- name: Audit UI dependencies
if: inputs.resume_tag == ''
working-directory: apps/ui
run: npm audit --audit-level=high
- name: Build UI dist
if: inputs.resume_tag == ''
working-directory: apps/ui
run: npm run build
- name: Install Playwright browser
if: inputs.validation_level == 'feature'
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
working-directory: apps/ui
run: npx playwright install --with-deps chromium
- name: Run release end-to-end tests
if: inputs.validation_level == 'feature'
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
working-directory: apps/ui
run: |
mkdir -p ../../.tmp
@@ -153,7 +213,7 @@ jobs:
PLAYWRIGHT_JSON_OUTPUT=../../.tmp/openapi-playwright.json npm run e2e
- name: Collect sanitized OpenAPI UI release evidence
if: inputs.validation_level == 'feature'
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
working-directory: apps/ui
run: |
trap 'rm -f ../../.tmp/openapi-playwright.json' EXIT
@@ -175,12 +235,14 @@ jobs:
--require-accepted
- name: Validate deployment manifests
if: inputs.resume_tag == ''
run: |
docker compose -f docker-compose.yml --env-file .env.example config -q
docker compose -f deploy/community/docker-compose.yml --env-file deploy/community/.env.example config -q
docker compose -f deploy/community/docker-compose.images.yml --env-file deploy/community/.env.images.example --profile local-db config -q
- name: Package release artifacts
if: inputs.resume_tag == ''
run: |
set -euo pipefail
mkdir -p dist/release
@@ -204,13 +266,14 @@ jobs:
sha256sum "${files[@]}" > "dist/crank-community-${IMAGE_TAG}-checksums.txt"
- name: Generate SBOM
if: inputs.resume_tag == ''
run: |
docker run --rm -v "$PWD:/workspace" anchore/syft:latest \
dir:/workspace -o spdx-json \
> dist/crank-community-${IMAGE_TAG}-sbom.spdx.json
- name: Publish release bundles, checksums, evidence, and SBOM to Gitea
if: startsWith(gitea.ref, 'refs/tags/v')
if: inputs.resume_tag == '' && startsWith(gitea.ref, 'refs/tags/v')
env:
# gitea.server_url inside the job container is the internal http://gitea:3000 origin;
# the publisher only forwards its token to the public HTTPS origin.
@@ -249,6 +312,8 @@ jobs:
docker login '${{ env.REGISTRY }}' -u "$DEPLOY_REGISTRY_USER" --password-stdin
- name: Build and push release images
env:
RELEASE_IMAGE_SCANNER: ${{ steps.pin_resume_scanner.outputs.path || 'scripts/scan-images.sh' }}
run: |
docker build -f apps/admin-api/Dockerfile \
-t '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' \
@@ -285,7 +350,7 @@ jobs:
--env-file .tmp/release-migration-smoke.env --profile local-db up -d --wait
docker compose -f deploy/community/docker-compose.images.yml \
--env-file .tmp/release-migration-smoke.env --profile local-db logs migrate | grep '"status":"applied"'
scripts/scan-images.sh \
"$RELEASE_IMAGE_SCANNER" \
'${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' \
'${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}' \
'${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}'
@@ -297,5 +362,6 @@ jobs:
docker push '${{ env.UI_IMAGE }}:latest'
- name: Show release bundle summary
if: inputs.resume_tag == ''
run: |
ls -lh dist
Generated
+215 -65
View File
@@ -25,7 +25,7 @@ dependencies = [
"async-trait",
"axum",
"axum-extra",
"base64",
"base64 0.22.1",
"crank-artifacts",
"crank-community-auth",
"crank-community-mcp",
@@ -186,7 +186,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -197,7 +197,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -349,6 +349,12 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "base64"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
[[package]]
name = "base64ct"
version = "1.8.3"
@@ -398,7 +404,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4"
dependencies = [
"async-stream",
"base64",
"base64 0.22.1",
"bitflags",
"bollard-buildkit-proto",
"bollard-stubs",
@@ -455,7 +461,7 @@ version = "1.52.1-rc.29.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f0a8ca8799131c1837d1282c3f81f31e76ceb0ce426e04a7fe1ccee3287c066"
dependencies = [
"base64",
"base64 0.22.1",
"bollard-buildkit-proto",
"bytes",
"prost",
@@ -584,6 +590,12 @@ dependencies = [
"tokio-util",
]
[[package]]
name = "const-oid"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be95d4a884358a21a199689323c29e9256b4a1b315e22f183d155422041d3a9a"
[[package]]
name = "cookie"
version = "0.18.1"
@@ -653,7 +665,7 @@ version = "0.3.1"
dependencies = [
"async-trait",
"axum",
"base64",
"base64 0.22.1",
"crank-core",
"crank-metrics",
"crank-trace",
@@ -688,7 +700,7 @@ dependencies = [
"argon2",
"async-trait",
"axum-extra",
"base64",
"base64 0.22.1",
"crank-core",
"crank-registry",
"rand 0.10.2",
@@ -705,7 +717,7 @@ version = "0.3.1"
dependencies = [
"async-trait",
"axum",
"base64",
"base64 0.22.1",
"crank-adapter-rest",
"crank-artifacts",
"crank-core",
@@ -722,6 +734,7 @@ dependencies = [
"opentelemetry",
"opentelemetry_sdk",
"reqwest 0.12.28",
"rmcp",
"serde",
"serde_json",
"sha2 0.10.9",
@@ -850,7 +863,7 @@ dependencies = [
"aes-gcm",
"async-trait",
"axum",
"base64",
"base64 0.22.1",
"crank-adapter-rest",
"crank-core",
"crank-import",
@@ -992,8 +1005,18 @@ version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
dependencies = [
"darling_core",
"darling_macro",
"darling_core 0.23.0",
"darling_macro 0.23.0",
]
[[package]]
name = "darling"
version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec"
dependencies = [
"darling_core 0.24.1",
"darling_macro 0.24.1",
]
[[package]]
@@ -1006,7 +1029,20 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
"syn",
"syn 2.0.118",
]
[[package]]
name = "darling_core"
version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff"
dependencies = [
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn 3.0.9",
]
[[package]]
@@ -1015,9 +1051,20 @@ version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
dependencies = [
"darling_core",
"darling_core 0.23.0",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
name = "darling_macro"
version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785"
dependencies = [
"darling_core 0.24.1",
"quote",
"syn 3.0.9",
]
[[package]]
@@ -1057,6 +1104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
dependencies = [
"block-buffer 0.12.1",
"const-oid",
"crypto-common 0.2.2",
"ctutils",
]
@@ -1069,7 +1117,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -1078,7 +1126,7 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d"
dependencies = [
"base64",
"base64 0.22.1",
"serde",
"serde_json",
]
@@ -1147,7 +1195,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -1309,7 +1357,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -1661,7 +1709,7 @@ version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
dependencies = [
"base64",
"base64 0.22.1",
"bytes",
"futures-channel",
"futures-util",
@@ -1672,7 +1720,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.5.10",
"socket2 0.6.4",
"tokio",
"tower-service",
"tracing",
@@ -1920,7 +1968,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
dependencies = [
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -2056,7 +2104,7 @@ version = "0.3.1"
dependencies = [
"async-trait",
"axum",
"base64",
"base64 0.22.1",
"crank-artifacts",
"crank-community-mcp",
"crank-config",
@@ -2121,7 +2169,7 @@ version = "0.18.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108"
dependencies = [
"base64",
"base64 0.22.1",
"evmap",
"indexmap 2.14.0",
"metrics",
@@ -2457,7 +2505,7 @@ dependencies = [
"regex",
"regex-syntax",
"structmeta",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -2471,6 +2519,12 @@ dependencies = [
"subtle",
]
[[package]]
name = "pastey"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -2494,7 +2548,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -2580,7 +2634,7 @@ dependencies = [
"itertools",
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -2636,7 +2690,7 @@ dependencies = [
"quinn-udp",
"rustc-hash",
"rustls",
"socket2 0.5.10",
"socket2 0.6.4",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -2675,9 +2729,9 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.5.10",
"socket2 0.6.4",
"tracing",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -2852,7 +2906,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -2890,7 +2944,7 @@ version = "0.12.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
dependencies = [
"base64",
"base64 0.22.1",
"bytes",
"cookie",
"cookie_store",
@@ -2934,7 +2988,7 @@ version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
dependencies = [
"base64",
"base64 0.22.1",
"bytes",
"futures-channel",
"futures-core",
@@ -2981,6 +3035,53 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rmcp"
version = "3.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d24a7e92f4bcaa36b97fa93c4fa7ca8f3e669eced949345b3462dba333c7f909"
dependencies = [
"async-trait",
"base64 0.23.1",
"bytes",
"chrono",
"futures",
"http",
"http-body",
"http-body-util",
"indexmap 2.14.0",
"libc",
"pastey",
"pin-project-lite",
"rand 0.10.2",
"rmcp-macros",
"schemars 1.2.1",
"serde",
"serde_json",
"sha2 0.11.0",
"sse-stream",
"thiserror 2.0.18",
"tokio",
"tokio-stream",
"tokio-util",
"tower-service",
"tracing",
"uuid",
]
[[package]]
name = "rmcp-macros"
version = "3.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3140f24a766f80f9ec1607bde85680959a69b83d46fa9ce87dadab7e67ab029"
dependencies = [
"darling 0.24.1",
"proc-macro2",
"quote",
"serde_json",
"syn 3.0.9",
]
[[package]]
name = "rustc-demangle"
version = "0.1.28"
@@ -3008,15 +3109,15 @@ dependencies = [
[[package]]
name = "rustix"
version = "1.1.4"
version = "1.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys 0.12.1",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3075,7 +3176,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3144,12 +3245,26 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc"
dependencies = [
"chrono",
"dyn-clone",
"ref-cast",
"schemars_derive",
"serde",
"serde_json",
]
[[package]]
name = "schemars_derive"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f"
dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals",
"syn 2.0.118",
]
[[package]]
name = "scoped-tls"
version = "1.0.1"
@@ -3293,7 +3408,18 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
name = "serde_derive_internals"
version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
]
[[package]]
@@ -3328,7 +3454,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -3349,7 +3475,7 @@ version = "3.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
dependencies = [
"base64",
"base64 0.22.1",
"bs58",
"chrono",
"hex",
@@ -3369,10 +3495,10 @@ version = "3.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
dependencies = [
"darling",
"darling 0.23.0",
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -3410,7 +3536,7 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -3544,7 +3670,7 @@ dependencies = [
name = "sqlx-core"
version = "0.9.0"
dependencies = [
"base64",
"base64 0.22.1",
"bytes",
"cfg-if",
"crc",
@@ -3586,7 +3712,7 @@ dependencies = [
"quote",
"sqlx-core",
"sqlx-macros-core",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -3609,7 +3735,7 @@ dependencies = [
"sqlx-mysql",
"sqlx-postgres",
"sqlx-sqlite",
"syn",
"syn 2.0.118",
"thiserror 2.0.18",
"tokio",
"url",
@@ -3650,7 +3776,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e"
dependencies = [
"atoi",
"base64",
"base64 0.22.1",
"bitflags",
"byteorder",
"crc",
@@ -3706,6 +3832,19 @@ dependencies = [
"uuid",
]
[[package]]
name = "sse-stream"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c25ac7aff0abd1dbc474536e40416e1102c7dd9bfba0b9861c6d357f835dcfb4"
dependencies = [
"bytes",
"futures-util",
"http-body",
"http-body-util",
"pin-project-lite",
]
[[package]]
name = "stable_deref_trait"
version = "1.2.1"
@@ -3738,7 +3877,7 @@ dependencies = [
"proc-macro2",
"quote",
"structmeta-derive",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -3749,7 +3888,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -3769,6 +3908,17 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "syn"
version = "3.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "sync_wrapper"
version = "1.0.2"
@@ -3786,7 +3936,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -3855,7 +4005,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -3866,7 +4016,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -3957,7 +4107,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -4002,7 +4152,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
dependencies = [
"async-trait",
"axum",
"base64",
"base64 0.22.1",
"bytes",
"h2",
"http",
@@ -4103,7 +4253,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -4232,7 +4382,7 @@ version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0"
dependencies = [
"base64",
"base64 0.22.1",
"log",
"percent-encoding",
"rustls",
@@ -4248,7 +4398,7 @@ version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c"
dependencies = [
"base64",
"base64 0.22.1",
"http",
"httparse",
"log",
@@ -4385,7 +4535,7 @@ dependencies = [
"bumpalo",
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
"wasm-bindgen-shared",
]
@@ -4477,7 +4627,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -4507,7 +4657,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -4518,7 +4668,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -4712,7 +4862,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
dependencies = [
"libc",
"rustix 1.1.4",
"rustix 1.1.6",
]
[[package]]
@@ -4734,7 +4884,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
"synstructure",
]
@@ -4755,7 +4905,7 @@ checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
@@ -4775,7 +4925,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
"synstructure",
]
@@ -4815,7 +4965,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.118",
]
[[package]]
+99 -87
View File
@@ -2,6 +2,10 @@
mod admin_auth_command;
#[path = "crank_migrate/db_connect.rs"]
mod db_connect;
#[path = "crank_migrate/reporting.rs"]
mod reporting;
use reporting::{active_rotation, ensure_no_options, read_required_key_file, rotation_json};
use crank_config::{ConfigSource, parse_migrator};
use crank_registry::{
@@ -11,7 +15,7 @@ use crank_registry::{
};
use crank_runtime::SecretCrypto;
use serde_json::json;
use std::{path::Path, process::ExitCode};
use std::process::ExitCode;
use time::OffsetDateTime;
const MASTER_KEY_ROTATION_PAGE_SIZE: i64 = 1_000;
#[tokio::main]
@@ -765,6 +769,59 @@ async fn process_rotation_batch(
processed += 1;
}
}
let mut after_task_id: Option<String> = None;
loop {
let tasks = registry
.list_tasks_for_master_key_epoch_page(
rotation.source_epoch,
after_task_id.as_deref(),
MASTER_KEY_ROTATION_PAGE_SIZE,
)
.await
.map_err(CliError::from_registry)?;
if tasks.is_empty() {
break;
}
for task in tasks {
after_task_id = Some(task.task_id.clone());
if task.target_epoch == Some(rotation.target_epoch) {
continue;
}
if max_versions.is_some_and(|limit| processed as usize >= limit) {
return Ok(processed);
}
let plaintext = current_crypto
.decrypt_for_epoch(&task.key_version, task.epoch, &task.ciphertext)
.map_err(|_| {
CliError::new(
"master_key_rotation_verification_failed",
"master_key.rotation",
"rerun_rotation_or_abort",
)
})?;
let target_ciphertext = target_crypto.encrypt(&plaintext).map_err(|_| {
CliError::new(
"master_key_rotation_verification_failed",
"master_key.rotation",
"rerun_rotation_or_abort",
)
})?;
let now = OffsetDateTime::now_utc();
registry
.stage_master_key_rotation_task_ciphertext(
&rotation.id,
&task.task_id,
rotation.source_epoch,
&target_ciphertext,
target_crypto.key_version(),
rotation.target_epoch,
&now,
)
.await
.map_err(CliError::from_registry)?;
processed += 1;
}
}
Ok(processed)
}
@@ -818,6 +875,47 @@ async fn verify_staged_targets(
verified += 1;
}
}
let mut after_task_id: Option<String> = None;
loop {
let tasks = registry
.list_target_tasks_for_master_key_rotation_page(
rotation.target_epoch,
after_task_id.as_deref(),
MASTER_KEY_ROTATION_PAGE_SIZE,
)
.await
.map_err(CliError::from_registry)?;
if tasks.is_empty() {
break;
}
for task in tasks {
after_task_id = Some(task.task_id.clone());
let ciphertext = task.target_ciphertext.as_deref().ok_or_else(|| {
CliError::new(
"master_key_rotation_verification_failed",
"master_key.rotation",
"rerun_rotation_or_abort",
)
})?;
let key_version = task.target_key_version.as_deref().ok_or_else(|| {
CliError::new(
"master_key_rotation_verification_failed",
"master_key.rotation",
"rerun_rotation_or_abort",
)
})?;
target_crypto
.decrypt_for_epoch(key_version, rotation.target_epoch, ciphertext)
.map_err(|_| {
CliError::new(
"master_key_rotation_verification_failed",
"master_key.rotation",
"rerun_rotation_or_abort",
)
})?;
verified += 1;
}
}
if verified != rotation.total_secret_versions {
return Err(CliError::new(
"master_key_rotation_verification_failed",
@@ -877,89 +975,3 @@ fn decrypt_current(
)
})
}
async fn active_rotation(
registry: &PostgresRegistry,
allowed_states: &[&str],
) -> Result<Option<MasterKeyRotationRecord>, CliError> {
let status = registry
.master_key_rotation_status()
.await
.map_err(CliError::from_registry)?;
Ok(status
.rotations
.into_iter()
.find(|rotation| allowed_states.contains(&rotation.state.as_str())))
}
fn rotation_json(rotation: &MasterKeyRotationRecord) -> serde_json::Value {
json!({
"id": rotation.id,
"source_epoch": rotation.source_epoch,
"target_epoch": rotation.target_epoch,
"state": rotation.state,
"backup_ref": rotation.backup_ref.as_ref().map(|_| "configured"),
"checkpoint_secret_id": rotation.checkpoint_secret_id,
"total_secret_versions": rotation.total_secret_versions,
"processed_secret_versions": rotation.processed_secret_versions,
"verified_secret_versions": rotation.verified_secret_versions,
"failure_code": rotation.failure_code,
})
}
fn read_required_key_file(path: Option<&str>) -> Result<String, CliError> {
let path = path.ok_or_else(|| {
CliError::new("invalid_command", "cli.arguments", "run_master_key_status")
})?;
if path.len() > 512 || path.bytes().any(|byte| byte.is_ascii_control()) {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
let metadata = std::fs::metadata(Path::new(path)).map_err(|_| {
CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
)
})?;
if !metadata.is_file() || metadata.len() > 16_384 {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
let value = std::fs::read_to_string(Path::new(path)).map_err(|_| {
CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
)
})?;
if value.trim().is_empty() {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
Ok(value)
}
fn ensure_no_options(options: &MasterKeyOptions) -> Result<(), CliError> {
if options.current_key_file.is_some()
|| options.target_key_file.is_some()
|| options.backup_ref.is_some()
|| options.rotation_id.is_some()
|| options.max_versions.is_some()
{
return Err(CliError::new(
"invalid_command",
"cli.arguments",
"run_master_key_status",
));
}
Ok(())
}
@@ -0,0 +1,98 @@
//! Leaf/reporting helpers of the migrator CLI.
//!
//! Extracted from `crank-migrate.rs` without behaviour changes so the binary entry stays within
//! the code-health line limit.
use std::path::Path;
use crank_registry::{MasterKeyRotationRecord, PostgresRegistry};
use serde_json::json;
use super::{CliError, MasterKeyOptions};
pub(super) async fn active_rotation(
registry: &PostgresRegistry,
allowed_states: &[&str],
) -> Result<Option<MasterKeyRotationRecord>, CliError> {
let status = registry
.master_key_rotation_status()
.await
.map_err(CliError::from_registry)?;
Ok(status
.rotations
.into_iter()
.find(|rotation| allowed_states.contains(&rotation.state.as_str())))
}
pub(super) fn rotation_json(rotation: &MasterKeyRotationRecord) -> serde_json::Value {
json!({
"id": rotation.id,
"source_epoch": rotation.source_epoch,
"target_epoch": rotation.target_epoch,
"state": rotation.state,
"backup_ref": rotation.backup_ref.as_ref().map(|_| "configured"),
"checkpoint_secret_id": rotation.checkpoint_secret_id,
"total_secret_versions": rotation.total_secret_versions,
"processed_secret_versions": rotation.processed_secret_versions,
"verified_secret_versions": rotation.verified_secret_versions,
"failure_code": rotation.failure_code,
})
}
pub(super) fn read_required_key_file(path: Option<&str>) -> Result<String, CliError> {
let path = path.ok_or_else(|| {
CliError::new("invalid_command", "cli.arguments", "run_master_key_status")
})?;
if path.len() > 512 || path.bytes().any(|byte| byte.is_ascii_control()) {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
let metadata = std::fs::metadata(Path::new(path)).map_err(|_| {
CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
)
})?;
if !metadata.is_file() || metadata.len() > 16_384 {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
let value = std::fs::read_to_string(Path::new(path)).map_err(|_| {
CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
)
})?;
if value.trim().is_empty() {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
Ok(value)
}
pub(super) fn ensure_no_options(options: &MasterKeyOptions) -> Result<(), CliError> {
if options.current_key_file.is_some()
|| options.target_key_file.is_some()
|| options.backup_ref.is_some()
|| options.rotation_id.is_some()
|| options.max_versions.is_some()
{
return Err(CliError::new(
"invalid_command",
"cli.arguments",
"run_master_key_status",
));
}
Ok(())
}
+2 -1
View File
@@ -29,7 +29,7 @@ sha2.workspace = true
sqlx.workspace = true
thiserror.workspace = true
time.workspace = true
tokio = { workspace = true, features = ["sync"] }
tokio = { workspace = true, features = ["sync", "signal"] }
tracing.workspace = true
tracing-subscriber.workspace = true
uuid.workspace = true
@@ -45,5 +45,6 @@ opentelemetry_sdk.workspace = true
prost.workspace = true
reqwest.workspace = true
tower.workspace = true
tokio = { workspace = true, features = ["net", "time"] }
tracing-opentelemetry.workspace = true
metrics-util = "0.20.4"
+149 -13
View File
@@ -201,20 +201,45 @@ async fn run(
// admin-api publishes into. The compatibility constructor keeps an unprovisioned/unsafe root
// fail-closed per read instead of preventing startup; nothing is ever written from this process.
let artifact_store = crank_artifacts::ArtifactStore::new(config.storage_root.clone());
// Story 4.2: the durable Task application entry shares the registry pool, the published catalog
// refresh interval and the exact admission limits of the runtime; the public wire stays
// synchronous/unsupported and no worker is started.
let task_store = PostgresTaskStore::new(
// Story 4.2/4.3: the durable Task application entry shares the registry pool, the published
// catalog refresh interval and the exact admission limits of the runtime; the public wire stays
// synchronous/unsupported while a fenced in-process worker executes accepted Tasks.
let task_store = std::sync::Arc::new(PostgresTaskStore::new(
registry.pool().clone(),
admission_limits(runtime_limits),
crank_core::AdmissionSchedulerConfig::default(),
)?;
)?);
let tasks = std::sync::Arc::new(crank_community_mcp::TaskApplication::new(
registry.clone(),
Duration::from_millis(config.refresh_ms),
cache_stores.coordination.clone(),
crank_runtime::TaskService::new(std::sync::Arc::new(task_store)),
crank_runtime::TaskService::with_secret_crypto(
task_store.clone(),
std::sync::Arc::new(secret_crypto.clone()),
),
));
// Story 4.3: the durable Task worker runs in this process over the same PostgreSQL store,
// published catalog, executor and encrypted-input authority as the accepting application.
let task_handler =
std::sync::Arc::new(crank_community_mcp::TaskWorkerHandler::from_application(
&tasks,
registry.clone(),
runtime.clone(),
secret_crypto.clone(),
));
let task_worker = std::sync::Arc::new(crank_runtime::TaskWorker::new(
task_store,
task_handler,
crank_runtime::TaskWorkerConfig {
owner: format!("mcp-server-{}", std::process::id()),
..crank_runtime::TaskWorkerConfig::default()
},
)?);
let (task_shutdown_tx, task_shutdown_rx) = tokio::sync::watch::channel(false);
let task_worker_handle = tokio::spawn({
let worker = task_worker.clone();
async move { worker.run_until_shutdown(task_shutdown_rx).await }
});
let app = build_app_with_artifacts_and_tasks(
registry,
Duration::from_millis(config.refresh_ms),
@@ -252,15 +277,77 @@ async fn run(
info!(name: "mcp.server.listening", bind_address = %config.bind_addr, "mcp-server listening");
*startup_completed = true;
if let Some(metrics_server) = metrics_server {
tokio::select! {
result = axum::serve(listener, app) => result?,
result = metrics_server.serve() => result?,
let serve_result: Result<(), Box<dyn std::error::Error>> = async {
if let Some(metrics_server) = metrics_server {
tokio::select! {
result = serve_with_worker_shutdown(app, listener, shutdown_signal(), task_shutdown_tx.clone()) => result?,
result = metrics_server.serve() => result?,
}
} else {
serve_with_worker_shutdown(app, listener, shutdown_signal(), task_shutdown_tx.clone())
.await?;
}
} else {
axum::serve(listener, app).await?;
Ok(())
}
.await;
// Stop new claims; active attempts checkpoint/drain or are cancelled inside the bounded worker
// shutdown timeout (<=30s), and queued PostgreSQL work is left for recovery.
task_shutdown_tx.send(true).ok();
if tokio::time::timeout(Duration::from_secs(30), task_worker_handle)
.await
.is_err()
{
tracing::warn!(
name: "mcp.task_worker.shutdown_timeout",
"durable Task worker did not stop within the story shutdown budget"
);
}
serve_result
}
/// Gracefully stop the durable Task worker as soon as shutdown is requested, while still
/// draining in-flight HTTP connections before returning.
async fn serve_with_worker_shutdown<S>(
app: axum::Router,
listener: tokio::net::TcpListener,
signal: S,
worker_stop: tokio::sync::watch::Sender<bool>,
) -> std::io::Result<()>
where
S: std::future::Future<Output = ()> + Send + 'static,
{
let worker_stop_graceful = worker_stop.clone();
let graceful = async move {
signal.await;
let _ = worker_stop_graceful.send(true);
};
let result = axum::serve(listener, app)
.with_graceful_shutdown(graceful)
.await;
let _ = worker_stop.send(true);
result
}
/// Process shutdown trigger for the HTTP surface and the durable Task worker.
async fn shutdown_signal() {
let ctrl_c = async {
let _ = tokio::signal::ctrl_c().await;
};
#[cfg(unix)]
let terminate = async {
if let Ok(mut signal) =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
{
signal.recv().await;
}
};
#[cfg(not(unix))]
let terminate = std::future::pending::<()>();
tokio::select! {
_ = ctrl_c => {}
_ = terminate => {}
}
Ok(())
}
fn init_observability(
@@ -506,3 +593,52 @@ fn runtime_cache_config(
.map(|value| value.expose_secret().to_owned()),
)
}
#[cfg(test)]
mod tests {
use super::*;
use axum::{Router, routing::get};
use std::time::Duration;
use tokio::sync::{oneshot, watch};
async fn slow_handler() -> &'static str {
tokio::time::sleep(Duration::from_millis(600)).await;
"ok"
}
#[tokio::test]
async fn worker_stops_before_http_drain_completes() {
let app = Router::new().route("/slow", get(slow_handler));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let (stop_tx, mut stop_rx) = watch::channel(false);
let (signal_tx, signal_rx) = oneshot::channel::<()>();
let handle = tokio::spawn(serve_with_worker_shutdown(
app,
listener,
async {
let _ = signal_rx.await;
},
stop_tx,
));
let client = reqwest::Client::new();
let request = client.get(format!("http://{}/slow", addr));
let in_flight = tokio::spawn(async move { request.send().await });
tokio::time::sleep(Duration::from_millis(150)).await;
let _ = signal_tx.send(());
tokio::time::timeout(Duration::from_millis(250), stop_rx.changed())
.await
.expect("worker stop signal should arrive before HTTP drain")
.expect("watch changed");
assert!(*stop_rx.borrow());
let response = in_flight.await.unwrap().unwrap();
assert!(response.status().is_success());
let serve_result = handle.await.unwrap();
assert!(serve_result.is_ok());
}
}
@@ -0,0 +1,555 @@
//! Story 4.4 ATDD: no outbound send exists without a committed durable attempt, and an elapsed
//! accepted deadline fails before dispatch.
//!
//! Both oracles drive the production composition (application entry → PG store → runtime executor →
//! real local HTTP upstream); only the upstream and the checkpoint failure are test-controlled.
use super::*;
use axum::Router;
use axum::http::StatusCode;
use axum::routing::post;
use crank_community_mcp::TaskWorkerHandler;
use crank_core::{
AgentOperationBinding, PlatformApiKeyScope, TaskState, ToolExecutionMode, ToolExecutionPolicy,
};
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use time::OffsetDateTime;
async fn composition(
registry: &PostgresRegistry,
upstream: &str,
policy: ToolExecutionPolicy,
caller_name: &str,
scoped_key: &str,
) -> (
Arc<PostgresTaskStore>,
crank_community_mcp::TaskApplication,
SecretCrypto,
crank_community_mcp::TaskPrincipal,
) {
let mut operation = common::test_operation(upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(policy);
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(registry, &operation.id, 1).await;
common::publish_agent_with_bindings(
registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(
registry,
caller_name,
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("test master-key identity registered");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let tasks = TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
);
let _ = scoped_key;
(store, tasks, crypto, caller)
}
fn worker_for(
registry: &PostgresRegistry,
store: &Arc<PostgresTaskStore>,
tasks: &TaskApplication,
crypto: &SecretCrypto,
owner: &str,
) -> TaskWorker {
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build();
let handler = Arc::new(TaskWorkerHandler::from_application(
tasks,
registry.clone(),
runtime,
crypto.clone(),
));
let config = TaskWorkerConfig {
owner: owner.to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
};
TaskWorker::new(store.clone(), handler, config).expect("valid worker")
}
async fn state_of(registry: &PostgresRegistry, task_id: crank_core::TaskId) -> Option<String> {
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap()
}
/// AC1: a dispatch checkpoint that cannot commit produces zero outbound calls and no attempt row.
#[tokio::test]
async fn checkpoint_commit_failure_never_reaches_upstream() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let (store, tasks, crypto, caller) = composition(
&registry,
&upstream,
background_policy(),
"attempt-guards-checkpoint",
"guards-checkpoint-key",
)
.await;
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
"guards-checkpoint-key",
&json!({"email": "user@example.com"}),
&unsampled_correlation(),
)
.await
.expect("durable acceptance");
// The checkpoint is the only statement that commits an attempt; it is refused here.
sqlx::raw_sql(sqlx::AssertSqlSafe(
"create function refuse_task_checkpoint() returns trigger language plpgsql as $$
begin
if new.current_attempt_id is not null and old.current_attempt_id is null then
raise exception 'durable checkpoint refused for test';
end if;
return new;
end $$;
create trigger refuse_task_checkpoint before update on tasks
for each row execute function refuse_task_checkpoint();",
))
.execute(registry.pool())
.await
.expect("checkpoint refusal trigger");
let worker = worker_for(
&registry,
&store,
&tasks,
&crypto,
"worker-guards-checkpoint",
);
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
tokio::time::sleep(Duration::from_secs(2)).await;
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(
calls.load(Ordering::SeqCst),
0,
"an uncommitted durable checkpoint must never produce an outbound call"
);
let attempts: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(
attempts, 0,
"no attempt history without a committed checkpoint"
);
assert_ne!(
state_of(&registry, acceptance.task_id).await.as_deref(),
Some("completed"),
"the refused attempt cannot complete the Task"
);
sqlx::raw_sql(sqlx::AssertSqlSafe(
"drop trigger refuse_task_checkpoint on tasks; drop function refuse_task_checkpoint();",
))
.execute(registry.pool())
.await
.expect("checkpoint refusal trigger removed");
server.abort();
}
/// AC6: an accepted absolute deadline that already elapsed fails the Task before any dispatch.
#[tokio::test]
async fn elapsed_deadline_fails_before_dispatch_without_upstream_call() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let short_deadline = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 300,
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
};
let (store, tasks, crypto, caller) = composition(
&registry,
&upstream,
short_deadline,
"attempt-guards-deadline",
"guards-deadline-key",
)
.await;
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
"guards-deadline-key",
&json!({"email": "user@example.com"}),
&unsampled_correlation(),
)
.await
.expect("durable acceptance");
// The accepted deadline elapses before any worker claims the Task.
tokio::time::sleep(Duration::from_millis(500)).await;
let worker = worker_for(&registry, &store, &tasks, &crypto, "worker-guards-deadline");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let mut state = None;
for _ in 0..100 {
tokio::time::sleep(Duration::from_millis(50)).await;
state = state_of(&registry, acceptance.task_id).await;
if state.as_deref() == Some("failed") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(state.as_deref(), Some("failed"));
assert_eq!(
calls.load(Ordering::SeqCst),
0,
"an elapsed deadline must fail before dispatch"
);
let attempts: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts, 0, "no attempt is checkpointed after the deadline");
let code: Option<String> =
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(code.as_deref(), Some("task_deadline_elapsed"));
assert_eq!(
sqlx::query_scalar::<_, String>("select state from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap(),
TaskState::Failed.as_str()
);
server.abort();
}
/// AC2: a write whose upstream idempotency is backed by server-owned evidence is retried with the
/// same stable key, in a bounded number of distinct durable attempts, and applies exactly one effect.
#[tokio::test]
async fn verified_idempotent_write_retries_with_stable_key_and_one_effect() {
use crank_community_mcp::VerifiedIdempotentWrite;
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
use std::collections::HashSet;
use std::sync::Mutex;
let registry = common::test_registry().await;
let requests = Arc::new(AtomicUsize::new(0));
let effects = Arc::new(AtomicUsize::new(0));
let keys = Arc::new(Mutex::new(Vec::<String>::new()));
let seen = Arc::new(Mutex::new(HashSet::<String>::new()));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let requests = Arc::clone(&requests);
let effects = Arc::clone(&effects);
let keys = Arc::clone(&keys);
let seen = Arc::clone(&seen);
move |headers: axum::http::HeaderMap| {
let requests = Arc::clone(&requests);
let effects = Arc::clone(&effects);
let keys = Arc::clone(&keys);
let seen = Arc::clone(&seen);
async move {
let key = headers
.get("idempotency-key")
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_owned();
keys.lock().expect("keys").push(key.clone());
let first_application = seen.lock().expect("seen").insert(key);
if first_application {
effects.fetch_add(1, Ordering::SeqCst);
}
let attempt = requests.fetch_add(1, Ordering::SeqCst);
if attempt == 0 {
// The first application is stored but its acknowledgement is lost.
return (
StatusCode::INTERNAL_SERVER_ERROR,
axum::Json(json!({"error": "acknowledgement lost"})),
);
}
(
StatusCode::OK,
axum::Json(json!({"id": "lead-1", "deduped": !first_application})),
)
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(background_policy());
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
operation.execution_config.idempotency = Some(IdempotencyPolicy {
mode: IdempotencyMode::Required,
ttl_ms: 60_000,
input_field: Some("idempotency_key".to_owned()),
header_name: Some("Idempotency-Key".to_owned()),
});
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(&registry, &operation.id, 1).await;
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(
&registry,
"attempt-guards-verified",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("test master-key identity registered");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let tasks = TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
);
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
"guards-verified-key",
&json!({"email": "user@example.com", "idempotency_key": "k-1"}),
&unsampled_correlation(),
)
.await
.expect("durable acceptance");
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
.build();
let handler = Arc::new(
TaskWorkerHandler::from_application(&tasks, registry.clone(), runtime, crypto.clone())
.with_verified_writes(vec![VerifiedIdempotentWrite {
operation_id: operation.id.as_str().to_owned(),
operation_version: 1,
key_header: "Idempotency-Key".to_owned(),
validity_ms: 60_000,
}]),
);
let config = TaskWorkerConfig {
owner: "worker-guards-verified".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let mut state = None;
for _ in 0..200 {
tokio::time::sleep(Duration::from_millis(50)).await;
state = state_of(&registry, acceptance.task_id).await;
if state.as_deref() == Some("completed") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(state.as_deref(), Some("completed"));
assert_eq!(
requests.load(Ordering::SeqCst),
2,
"exactly one retry of the write"
);
assert_eq!(
effects.load(Ordering::SeqCst),
1,
"the upstream applied the write exactly once"
);
assert_eq!(
keys.lock().expect("keys").as_slice(),
["k-1".to_owned(), "k-1".to_owned()],
"the same stable key is delivered on both attempts"
);
let attempts = store
.attempts(
&crank_core::TaskAccess {
workspace_id: common::test_workspace_id().to_string(),
owner_kind: crank_core::TaskOwnerKind::Agent,
owner_id: "agent_attempt_guards".to_owned(),
authorization_scope: String::new(),
},
acceptance.task_id,
)
.await;
// Owner scope of this principal is the agent of the published binding; read the history through
// the application entry instead of guessing the authorization scope string.
assert!(attempts.is_err(), "a foreign scope never reads the history");
let attempts = tasks
.attempts(&caller, TOOL_NAME, acceptance.task_id)
.await
.expect("owner-scoped attempt history");
assert_eq!(attempts.len(), 2, "two distinct durable attempts");
assert_eq!(attempts[0].ordinal, 1);
assert_eq!(attempts[1].ordinal, 2);
assert!(attempts.iter().all(|attempt| attempt.retry_allowed));
assert_eq!(
attempts[0].outcome,
Some(crank_core::AttemptOutcomeClass::RetryScheduled)
);
assert_eq!(
attempts[1].outcome,
Some(crank_core::AttemptOutcomeClass::Completed)
);
assert_eq!(
attempts[0].upstream_key_digest, attempts[1].upstream_key_digest,
"the same stable key digest is recorded on both attempts"
);
assert!(attempts[0].upstream_key_digest.is_some());
let record = tasks
.status(&caller, TOOL_NAME, acceptance.task_id)
.await
.expect("status");
assert_eq!(record.pins.retry.max_attempts, 2);
}
@@ -0,0 +1,432 @@
//! Story 4.3 ATDD: durable Attempt ID reaches the adapter attempt span (PF-1).
//!
//! End-to-end: a real durable Task worker dispatches a retrying GET through a real local
//! upstream, and the `adapter_attempt` spans carry the same durable Attempt ID that is
//! persisted in `tasks.current_attempt_id` on both attempts.
use super::super::common;
use axum::Router;
use axum::http::StatusCode;
use axum::routing::get;
use crank_community_mcp::TaskWorkerHandler;
use crank_core::{
AgentOperationBinding, CorrelationContext, RetryPolicy, Target, ToolExecutionPolicy,
};
use crank_runtime::{
OutboundHttpPolicy, SecretCrypto, TaskWorker, TaskWorkerConfig, community_with_outbound_policy,
};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use time::OffsetDateTime;
use tracing::{Id, Subscriber, field::Visit};
use tracing_subscriber::{Layer, layer::SubscriberExt, registry::LookupSpan};
const AGENT_SLUG: &str = "reports";
const TOOL_NAME: &str = "attempt_trace";
const KEY: &str = "attempt-trace-key-1";
async fn publish(
registry: &crank_registry::PostgresRegistry,
operation_id: &crank_core::OperationId,
version: u32,
) {
registry
.publish_operation(crank_registry::PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id,
version,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
}
async fn application_with_store(
registry: &crank_registry::PostgresRegistry,
) -> (
crank_community_mcp::TaskApplication,
std::sync::Arc<crank_registry::PostgresTaskStore>,
SecretCrypto,
) {
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("test master-key identity registered");
let store = std::sync::Arc::new(
crank_registry::PostgresTaskStore::new(
registry.pool().clone(),
crank_registry::AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let tasks = crank_community_mcp::TaskApplication::new(
registry.clone(),
std::time::Duration::ZERO,
std::sync::Arc::new(crank_runtime::InMemoryCoordinationStateStore::default()),
crank_runtime::TaskService::with_secret_crypto(
store.clone(),
std::sync::Arc::new(crypto.clone()),
),
);
(tasks, store, crypto)
}
async fn principal(
registry: &crank_registry::PostgresRegistry,
name: &str,
scopes: &[crank_core::PlatformApiKeyScope],
) -> crank_community_mcp::TaskPrincipal {
let token = common::create_platform_api_key(registry, AGENT_SLUG, name, scopes).await;
let secret_hash = common::hash_access_secret(&token);
let api_key = registry
.get_platform_api_key_by_secret_for_agent_slug(
common::test_workspace_slug(),
AGENT_SLUG,
&secret_hash,
)
.await
.unwrap()
.expect("an active platform key resolves");
registry
.touch_platform_api_key(
&api_key.api_key.workspace_id,
&api_key.api_key.id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
crank_community_mcp::TaskPrincipal::Agent {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
credential: crank_core::VerifiedMachineCredential {
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
max_security_level: crank_core::OperationSecurityLevel::Standard,
scopes: api_key.api_key.scopes.clone(),
platform_api_key_id: Some(api_key.api_key.id.clone()),
},
}
}
#[tokio::test]
async fn adapter_attempt_span_carries_durable_attempt_id() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
get({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
let n = calls.fetch_add(1, Ordering::SeqCst);
if n == 0 {
return (
StatusCode::INTERNAL_SERVER_ERROR,
axum::Json(serde_json::json!({"error":"boom"})),
);
}
(
StatusCode::OK,
axum::Json(serde_json::json!({"id":"lead-1"})),
)
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
let Target::Rest(target) = &mut operation.target;
target.method = crank_core::HttpMethod::Get;
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
mode: crank_core::ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
});
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(&registry, &operation.id, 1).await;
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(
&registry,
"attempt-trace",
&[
crank_core::PlatformApiKeyScope::Read,
crank_core::PlatformApiKeyScope::Write,
],
)
.await;
let (tasks, store, crypto) = application_with_store(&registry).await;
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&serde_json::json!({"email": "user@example.com"}),
&CorrelationContext::generate(),
)
.await
.expect("durable acceptance");
let runtime =
community_with_outbound_policy(OutboundHttpPolicy::allowing_hosts(["127.0.0.1"])).build();
let handler = Arc::new(TaskWorkerHandler::from_application(
&tasks,
registry.clone(),
runtime,
crypto,
));
let config = TaskWorkerConfig {
owner: "runtime-worker-attempt-trace".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: std::time::Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: std::time::Duration::from_secs(5),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let capture = TraceCapture::default();
let subscriber = tracing_subscriber::registry().with(capture.clone());
let _guard = tracing::subscriber::set_default(subscriber);
let worker_task = tokio::spawn(async move { worker.run().await });
let mut state = None;
for _ in 0..200 {
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let current: Option<String> =
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
state = current;
if state.as_deref() == Some("completed") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
tokio::time::timeout(std::time::Duration::from_secs(10), worker_task)
.await
.expect("worker stops inside the budget")
.expect("worker exits cleanly")
.expect("worker reports no error");
server.abort();
let durable_attempt_id: Option<String> = sqlx::query_scalar::<_, Option<String>>(
"select current_attempt_id from tasks where task_id = $1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(state.as_deref(), Some("completed"));
// Story 4.4: every outbound send is its own durable attempt, so the two retries of one lease carry
// two distinct attempt identities, both recorded in the immutable attempt history.
let attempt_rows: Vec<(String, i32)> = sqlx::query_as(
"select attempt_id, ordinal from task_attempts where task_id = $1 order by ordinal",
)
.bind(acceptance.task_id.as_uuid())
.fetch_all(registry.pool())
.await
.unwrap();
assert_eq!(attempt_rows.len(), 2, "two durable attempts were committed");
assert_eq!(attempt_rows[0].1, 1);
assert_eq!(attempt_rows[1].1, 2);
assert_eq!(
Some(attempt_rows[1].0.as_str()),
durable_attempt_id.as_deref(),
"the Task points at its latest durable attempt"
);
let spans = capture.snapshot();
let attempt_spans: Vec<_> = spans
.iter()
.filter(|s| s.name == "adapter_attempt")
.collect();
assert_eq!(attempt_spans.len(), 2, "two adapter_attempt spans");
for span in &attempt_spans {
let aid = span.fields.get("attempt_id").cloned().unwrap_or_default();
assert!(!aid.is_empty(), "attempt_id must not be empty");
assert!(
!aid.contains("traceparent"),
"attempt_id must not contain traceparent"
);
assert!(
attempt_rows.iter().any(|(id, _)| id == &aid),
"every attempt span names an attempt that was durably committed"
);
}
let ids: Vec<String> = attempt_spans
.iter()
.map(|s| s.fields.get("attempt_id").cloned().unwrap_or_default())
.collect();
assert_ne!(ids[0], ids[1], "each retry has its own durable attempt id");
assert_eq!(
spans
.iter()
.filter(|span| span.name == "durable_task_attempt")
.count(),
1,
"one durable attempt span per dispatch"
);
let runtime_span = spans
.iter()
.find(|span| span.name == "runtime.execute")
.expect("runtime.execute span");
assert_eq!(
runtime_span.parent_name,
Some("durable_task_attempt"),
"the execution continues the durable attempt trace"
);
for span in &attempt_spans {
assert_eq!(
span.parent_name,
Some("runtime.execute"),
"each retry is a child attempt span under the durable execution"
);
}
let attempts: Vec<String> = attempt_spans
.iter()
.map(|s| s.fields.get("attempt").cloned().unwrap_or_default())
.collect();
assert!(attempts.contains(&"1".to_string()));
assert!(attempts.contains(&"2".to_string()));
for span in &attempt_spans {
assert_eq!(
span.fields.get("max_attempts").cloned().unwrap_or_default(),
"2"
);
}
}
// TraceCapture minimal port from crates/crank-runtime/tests/stages.rs
#[derive(Clone, Default)]
struct TraceCapture {
spans: Arc<Mutex<Vec<CapturedSpan>>>,
}
impl TraceCapture {
fn snapshot(&self) -> Vec<CapturedSpan> {
self.spans.lock().expect("span lock").clone()
}
}
#[derive(Clone, Debug)]
struct CapturedSpan {
name: &'static str,
parent_name: Option<&'static str>,
fields: std::collections::BTreeMap<String, String>,
}
impl<S> Layer<S> for TraceCapture
where
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
{
fn on_new_span(
&self,
attributes: &tracing::span::Attributes<'_>,
id: &Id,
context: tracing_subscriber::layer::Context<'_, S>,
) {
let parent = attributes
.parent()
.and_then(|parent| context.span(parent))
.or_else(|| {
attributes
.is_contextual()
.then(|| context.lookup_current())
.flatten()
});
let mut visitor = FieldVisitor::default();
attributes.record(&mut visitor);
let mut spans = self.spans.lock().expect("span lock");
let index = spans.len();
spans.push(CapturedSpan {
name: attributes.metadata().name(),
parent_name: parent.map(|span| span.metadata().name()),
fields: visitor.fields,
});
context
.span(id)
.expect("span exists")
.extensions_mut()
.insert(index);
}
fn on_record(
&self,
id: &Id,
values: &tracing::span::Record<'_>,
context: tracing_subscriber::layer::Context<'_, S>,
) {
let mut visitor = FieldVisitor::default();
values.record(&mut visitor);
let span = context.span(id).expect("span exists");
let index = *span.extensions().get::<usize>().expect("capture index");
self.spans.lock().expect("span lock")[index]
.fields
.extend(visitor.fields);
}
}
#[derive(Default)]
struct FieldVisitor {
fields: std::collections::BTreeMap<String, String>,
}
impl Visit for FieldVisitor {
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
self.fields
.insert(field.name().to_owned(), value.to_owned());
}
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
self.fields
.insert(field.name().to_owned(), format!("{value:?}"));
}
}
@@ -0,0 +1,846 @@
//! ATDD Story 4.4 (Beam): cancel through TaskApplication -> PG -> worker -> upstream.
use super::common;
use super::unsampled_correlation;
use axum::Router;
use axum::http::StatusCode;
use axum::routing::post;
use crank_community_mcp::{TaskApplication, TaskWorkerHandler};
use crank_core::{
AgentOperationBinding, NoTasksFallback, PlatformApiKeyId, PlatformApiKeyScope, TaskState,
TaskStore, ToolExecutionMode, ToolExecutionPolicy, VerifiedMachineCredential,
};
use crank_registry::{AdmissionLimits, PostgresRegistry, PostgresTaskStore, PublishRequest};
use crank_runtime::{
InMemoryCoordinationStateStore, SecretCrypto, TaskService, TaskWorker, TaskWorkerConfig,
};
use serde_json::json;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use time::OffsetDateTime;
const AGENT_SLUG: &str = "reports";
const TOOL_NAME: &str = "report_gen";
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
async fn state_of(registry: &PostgresRegistry, task_id: crank_core::TaskId) -> Option<String> {
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap()
}
async fn seed_tool(registry: &PostgresRegistry, upstream: &str, policy: ToolExecutionPolicy) {
let mut operation = common::test_operation(upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(policy);
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
common::publish_agent_with_bindings(
registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
}
async fn principal(registry: &PostgresRegistry, name: &str) -> crank_community_mcp::TaskPrincipal {
let token = common::create_platform_api_key(
registry,
AGENT_SLUG,
name,
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let secret_hash = common::hash_access_secret(&token);
let api_key = registry
.get_platform_api_key_by_secret_for_agent_slug(
common::test_workspace_slug(),
AGENT_SLUG,
&secret_hash,
)
.await
.unwrap()
.expect("active key");
registry
.touch_platform_api_key(
&api_key.api_key.workspace_id,
&api_key.api_key.id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
crank_community_mcp::TaskPrincipal::Agent {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
credential: VerifiedMachineCredential {
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
max_security_level: crank_core::OperationSecurityLevel::Standard,
scopes: api_key.api_key.scopes.clone(),
platform_api_key_id: Some(api_key.api_key.id.clone()),
},
}
}
async fn application(registry: &PostgresRegistry) -> TaskApplication {
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store, Arc::new(crypto)),
)
}
#[tokio::test]
async fn cancel_during_active_request_is_observed_and_leaves_one_unknown_outcome() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let (tx, rx) = tokio::sync::oneshot::channel::<()>();
let signal = Arc::new(std::sync::Mutex::new(Some(tx)));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
let signal = Arc::clone(&signal);
move || {
let calls = Arc::clone(&calls);
let signal = Arc::clone(&signal);
async move {
calls.fetch_add(1, Ordering::SeqCst);
if let Some(tx) = signal.lock().unwrap().take() {
let _ = tx.send(());
}
tokio::time::sleep(Duration::from_millis(1500)).await;
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &upstream, policy).await;
let caller = principal(&registry, "cancel-s1").await;
let foreign = principal(&registry, "cancel-s1-foreign").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"s1-key",
&json!({"email":"u"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build();
let handler = Arc::new(TaskWorkerHandler::from_application(
&app,
registry.clone(),
runtime,
crypto.clone(),
));
let worker = TaskWorker::new(
store,
handler,
TaskWorkerConfig {
owner: "worker-s1".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
renewal_interval: Duration::from_millis(100),
shutdown_timeout: Duration::from_secs(5),
},
)
.expect("worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let _ = tokio::time::timeout(Duration::from_secs(5), rx).await;
let cancel = app
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
.await
.expect("cancel");
match cancel {
crank_core::TaskCancelOutcome::OutcomeUnknown(record) => {
assert_eq!(record.state, TaskState::OutcomeUnknown);
}
other => panic!("expected OutcomeUnknown got {other:?}"),
}
let foreign_cancel = app
.cancel(&foreign, TOOL_NAME, acceptance.task_id, None)
.await;
assert!(matches!(
foreign_cancel,
Err(crank_community_mcp::TaskApplicationError::NotFound)
));
tokio::time::sleep(Duration::from_millis(1600)).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(
state_of(&registry, acceptance.task_id).await.as_deref(),
Some("outcome_unknown")
);
let certainty: Option<String> = sqlx::query_scalar(
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
assert_eq!(certainty.as_deref(), Some("unknown"));
let code: Option<String> =
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(code.as_deref(), Some("cancelled_after_dispatch"));
let attempts_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts_count, 1);
let cancel2 = app
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
.await
.expect("cancel2");
assert!(matches!(
cancel2,
crank_core::TaskCancelOutcome::OutcomeUnknown(_)
));
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
}
#[tokio::test]
async fn cancel_before_any_dispatch_is_cancelled_without_upstream_call() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &upstream, policy).await;
let caller = principal(&registry, "cancel-s3").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"s3-key",
&json!({"email":"u"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let cancel = app
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
.await
.expect("cancel");
match cancel {
crank_core::TaskCancelOutcome::Cancelled(record) => {
assert_eq!(record.state, TaskState::Cancelled)
}
other => panic!("expected Cancelled got {other:?}"),
}
assert_eq!(
state_of(&registry, acceptance.task_id).await.as_deref(),
Some("cancelled")
);
let attempts = app
.attempts(&caller, TOOL_NAME, acceptance.task_id)
.await
.expect("attempts");
assert!(attempts.is_empty());
assert_eq!(calls.load(Ordering::SeqCst), 0);
server.abort();
}
#[tokio::test]
async fn cancel_during_pinned_backoff_never_redispatches_and_is_outcome_unknown() {
use crank_community_mcp::VerifiedIdempotentWrite;
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
let registry = common::test_registry().await;
let requests = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let requests = Arc::clone(&requests);
move |headers: axum::http::HeaderMap| {
let requests = Arc::clone(&requests);
async move {
let _ = headers.get("idempotency-key");
let attempt = requests.fetch_add(1, Ordering::SeqCst);
if attempt == 0 {
// Applied upstream, acknowledgement lost after the caller had time to observe that
// the request really left the worker.
tokio::time::sleep(Duration::from_millis(600)).await;
return (
StatusCode::INTERNAL_SERVER_ERROR,
axum::Json(json!({"error":"acknowledgement lost"})),
);
}
(StatusCode::OK, axum::Json(json!({"id":"lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
});
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
operation.execution_config.idempotency = Some(IdempotencyPolicy {
mode: IdempotencyMode::Required,
ttl_ms: 60_000,
input_field: Some("idempotency_key".to_owned()),
header_name: Some("Idempotency-Key".to_owned()),
});
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(&registry, "cancel-s2").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"s2-key",
&json!({"email":"u","idempotency_key":"k-2"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
.build();
let handler = Arc::new(
TaskWorkerHandler::from_application(&app, registry.clone(), runtime, crypto.clone())
.with_verified_writes(vec![VerifiedIdempotentWrite {
operation_id: operation.id.as_str().to_owned(),
operation_version: 1,
key_header: "Idempotency-Key".to_owned(),
validity_ms: 60_000,
}]),
);
// Reproduce deterministically the durable state the executor commits after an ambiguous attempt
// whose bounded retry is allowed: one attempt resolved with certainty=unknown, the retry armed and
// `current_attempt_id` cleared. The real retry window is a pinned millisecond backoff, so racing a
// running executor against the cancel would not be a repeatable oracle.
let lease = store
.claim_batch("worker-s2-prep", 1, 10_000)
.await
.expect("claim")
.pop()
.expect("one claimed lease");
let attempt = store
.checkpoint_dispatch(crank_core::BeginAttempt {
task_id: acceptance.task_id,
owner: lease.owner.clone(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: format!("attempt-{}", uuid::Uuid::now_v7()),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: true,
})
.await
.expect("durable checkpoint");
let resolution = store
.resolve_attempt(
&attempt,
crank_core::AttemptOutcome::Uncertain {
error: crank_core::TaskEnvelope::try_redacted(
&json!({"code": "upstream_ack_lost", "phase": "dispatched"}),
)
.expect("bounded attempt error"),
repeatable: true,
phase: crank_core::AttemptPhase::Dispatched,
},
)
.await
.expect("attempt resolution");
assert!(
matches!(resolution, crank_core::AttemptResolution::RetryAt { .. }),
"an ambiguous but safely repeatable attempt arms the pinned retry"
);
let cancel = app
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
.await
.expect("cancel");
match cancel {
crank_core::TaskCancelOutcome::OutcomeUnknown(record) => {
assert_eq!(record.state, crank_core::TaskState::OutcomeUnknown)
}
other => panic!("cancel after an ambiguous attempt must stay uncertain, got {other:?}"),
}
// A real worker must never redispatch or terminate the Task afterwards: the uncertain state is not
// claimable and no second terminal write is possible.
let worker = TaskWorker::new(
store.clone(),
handler,
TaskWorkerConfig {
owner: "worker-s2".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
},
)
.expect("worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
tokio::time::sleep(Duration::from_secs(1)).await;
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(
requests.load(Ordering::SeqCst),
0,
"no outbound send after the cancel"
);
assert_eq!(
state_of(&registry, acceptance.task_id).await.as_deref(),
Some("outcome_unknown")
);
let attempts_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts_count, 1, "no second durable attempt");
let certainty: Option<String> = sqlx::query_scalar(
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
assert_eq!(certainty.as_deref(), Some("unknown"));
let code: Option<String> =
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(code.as_deref(), Some("cancelled_after_dispatch"));
}
#[tokio::test]
async fn refused_attempt_after_lost_ack_keeps_outcome_unknown() {
use crank_community_mcp::VerifiedIdempotentWrite;
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
let registry = common::test_registry().await;
let requests = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let requests = Arc::clone(&requests);
move |headers: axum::http::HeaderMap| {
let requests = Arc::clone(&requests);
async move {
let _ = headers.get("idempotency-key");
let attempt = requests.fetch_add(1, Ordering::SeqCst);
if attempt == 0 {
// Applied upstream, acknowledgement lost after the caller had time to observe that
// the request really left the worker.
tokio::time::sleep(Duration::from_millis(600)).await;
return (
StatusCode::INTERNAL_SERVER_ERROR,
axum::Json(json!({"error":"acknowledgement lost"})),
);
}
(StatusCode::OK, axum::Json(json!({"id":"lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
});
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
operation.execution_config.idempotency = Some(IdempotencyPolicy {
mode: IdempotencyMode::Required,
ttl_ms: 60_000,
input_field: Some("idempotency_key".to_owned()),
header_name: Some("Idempotency-Key".to_owned()),
});
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(&registry, "cancel-s4").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"s4-key",
&json!({"email":"u","idempotency_key":"k-4"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
.build();
let handler = Arc::new(
TaskWorkerHandler::from_application(&app, registry.clone(), runtime, crypto.clone())
.with_verified_writes(vec![VerifiedIdempotentWrite {
operation_id: operation.id.as_str().to_owned(),
operation_version: 1,
key_header: "Idempotency-Key".to_owned(),
validity_ms: 60_000,
}]),
);
let worker = TaskWorker::new(
store.clone(),
handler,
TaskWorkerConfig {
owner: "worker-s4".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
},
)
.expect("worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
// Revoke the key the Task was accepted with while the first outbound write is provably in flight:
// that attempt still resolves as an uncertain write, and the next attempt is refused by the
// authority before its checkpoint.
for _ in 0..200 {
if requests.load(Ordering::SeqCst) >= 1 {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
assert_eq!(
requests.load(Ordering::SeqCst),
1,
"the first attempt is dispatched before the refusal"
);
registry
.revoke_platform_api_key_for_agent(
&common::test_workspace_id(),
&common::test_agent_id(AGENT_SLUG),
&PlatformApiKeyId::new("pk_cancel-s4".to_owned()),
&OffsetDateTime::now_utc(),
)
.await
.expect("revoke");
let mut state = None;
for _ in 0..200 {
tokio::time::sleep(Duration::from_millis(50)).await;
state = state_of(&registry, acceptance.task_id).await;
if state.as_deref() == Some("outcome_unknown") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(
state.as_deref(),
Some("outcome_unknown"),
"a refused attempt after a lost acknowledgement must stay uncertain"
);
assert_eq!(
requests.load(Ordering::SeqCst),
1,
"the refused attempt is never dispatched"
);
let attempts_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts_count, 1, "no second durable attempt");
let certainty: Option<String> = sqlx::query_scalar(
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
assert_eq!(certainty.as_deref(), Some("unknown"));
let phase: Option<String> = sqlx::query_scalar(
"select phase from task_attempts where task_id = $1 order by ordinal desc limit 1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
assert_eq!(phase.as_deref(), Some("dispatched"));
let error: Option<(Option<String>, Option<String>)> = sqlx::query_as(
"select error_envelope ->> 'code', error_envelope ->> 'phase' from tasks where task_id = $1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
let (code, error_phase) = error.expect("the refused Task carries its bounded error");
assert_eq!(code.as_deref(), Some("task_authority_rejected"));
assert_eq!(
error_phase.as_deref(),
Some("dispatched"),
"the error keeps the phase of the attempt that may have reached the upstream"
);
}
@@ -0,0 +1,965 @@
//! ATDD Story 4.5 (Beam): durable input continuation through TaskApplication -> PG -> worker -> upstream.
use super::common;
use super::unsampled_correlation;
use axum::Router;
use axum::http::StatusCode;
use axum::routing::post;
use crank_community_mcp::{TaskApplication, TaskWorkerHandler};
use crank_core::{
NoTasksFallback, PlatformApiKeyScope, TaskId, TaskState, ToolExecutionMode, ToolExecutionPolicy,
};
use crank_registry::{AdmissionLimits, PostgresRegistry, PostgresTaskStore, PublishRequest};
use crank_runtime::{
InMemoryCoordinationStateStore, SecretCrypto, TaskService, TaskWorker, TaskWorkerConfig,
};
use serde_json::json;
use sqlx::Row;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use time::OffsetDateTime;
const AGENT_SLUG: &str = "reports";
pub(crate) const TOOL_NAME: &str = "report_gen";
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
pub(crate) async fn state_of(registry: &PostgresRegistry, task_id: TaskId) -> Option<String> {
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap()
}
pub(crate) async fn wait_for_state(
registry: &PostgresRegistry,
task_id: TaskId,
desired: &str,
limit: Duration,
) -> bool {
let deadline = tokio::time::Instant::now() + limit;
while tokio::time::Instant::now() < deadline {
if state_of(registry, task_id).await.as_deref() == Some(desired) {
return true;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
state_of(registry, task_id).await.as_deref() == Some(desired)
}
pub(crate) async fn seed_tool(
registry: &PostgresRegistry,
upstream: &str,
policy: ToolExecutionPolicy,
) {
let mut operation = common::test_operation(upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(policy);
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
common::publish_agent_with_bindings(
registry,
AGENT_SLUG,
vec![crank_core::AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
}
pub(crate) async fn principal(
registry: &PostgresRegistry,
name: &str,
) -> crank_community_mcp::TaskPrincipal {
let token = common::create_platform_api_key(
registry,
AGENT_SLUG,
name,
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let secret_hash = common::hash_access_secret(&token);
let api_key = registry
.get_platform_api_key_by_secret_for_agent_slug(
common::test_workspace_slug(),
AGENT_SLUG,
&secret_hash,
)
.await
.unwrap()
.expect("active key");
registry
.touch_platform_api_key(
&api_key.api_key.workspace_id,
&api_key.api_key.id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
crank_community_mcp::TaskPrincipal::Agent {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
credential: crank_core::VerifiedMachineCredential {
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
max_security_level: crank_core::OperationSecurityLevel::Standard,
scopes: api_key.api_key.scopes.clone(),
platform_api_key_id: Some(api_key.api_key.id.clone()),
},
}
}
pub(crate) async fn application(registry: &PostgresRegistry) -> TaskApplication {
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store, Arc::new(crypto)),
)
}
pub(crate) fn spawn_worker(
registry: &PostgresRegistry,
app: &TaskApplication,
) -> (
Arc<std::sync::atomic::AtomicBool>,
tokio::task::JoinHandle<()>,
) {
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
let pool = registry.pool().clone();
let store = Arc::new(
PostgresTaskStore::new(
pool,
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build();
let handler = Arc::new(TaskWorkerHandler::from_application(
app,
registry.clone(),
runtime,
crypto,
));
let worker = TaskWorker::new(
store,
handler,
TaskWorkerConfig {
owner: "worker-input".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
renewal_interval: Duration::from_millis(100),
shutdown_timeout: Duration::from_secs(5),
},
)
.expect("worker");
let stop_flag = worker.stop_flag();
let jh = tokio::spawn(async move {
let _ = worker.run().await;
});
(stop_flag, jh)
}
#[tokio::test]
async fn input_continuation_happy_path_accept_update_resume_replay() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &upstream, policy).await;
let caller = principal(&registry, "input-happy").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"input-key-1",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("acceptance");
assert_eq!(acceptance.state, TaskState::Queued);
let task_id = acceptance.task_id;
let (stop, worker_task) = spawn_worker(&registry, &app);
assert!(
wait_for_state(
&registry,
task_id,
"input_required",
Duration::from_secs(10)
)
.await,
"worker must suspend for missing email"
);
// positive control: before update no outbound request
assert_eq!(calls.load(Ordering::SeqCst), 0);
// persisted request fields
let row = sqlx::query(
"select wait_id, expected_version, schema_snapshot, projected_schema, prompt_code, expires_at from task_input_requests where task_id = $1",
)
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
let wait_id: String = row.get("wait_id");
let expected_version: i32 = row.get("expected_version");
let schema_snapshot: serde_json::Value = row.get("schema_snapshot");
let projected_schema: serde_json::Value = row.get("projected_schema");
let prompt_code: String = row.get("prompt_code");
let _expires_at: OffsetDateTime = row.get("expires_at");
assert!(!wait_id.is_empty());
assert!(expected_version > 0);
assert!(schema_snapshot.is_object());
assert!(projected_schema.is_object());
assert_eq!(prompt_code, "required_input_missing");
let audit_count_before: i64 = sqlx::query_scalar(
"select count(*)::bigint from task_input_audit where task_id = $1 and event = 'input_required'",
)
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert!(audit_count_before >= 1);
// stop worker before update to avoid races
stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
let outcome = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version as u32,
"update-key-1",
&json!({"email": "user@example.com"}),
)
.await
.expect("valid update");
assert_eq!(outcome.state, TaskState::Working);
assert!(!outcome.replayed);
assert_eq!(outcome.task_id, task_id);
assert_eq!(outcome.wait_id, wait_id);
let (stop2, worker_task2) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_id, "completed", Duration::from_secs(10)).await,
"worker must complete after update"
);
stop2.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task2).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
let attempts = app
.attempts(&caller, TOOL_NAME, task_id)
.await
.expect("attempts");
assert_eq!(attempts.len(), 1);
let effective_fp: String =
sqlx::query_scalar("select effective_input_fingerprint from tasks where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts[0].input_fingerprint, effective_fp);
let input_fp: String =
sqlx::query_scalar("select input_fingerprint from tasks where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_ne!(effective_fp, input_fp);
// An identical answer replay still returns the committed outcome after the Task completed:
// no second transition, audit row or outbound call.
let completed_replay = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version as u32,
"update-key-1",
&json!({"email": "user@example.com"}),
)
.await
.expect("identical replay after completion");
assert!(completed_replay.replayed);
assert_eq!(completed_replay.version, outcome.version);
assert_eq!(calls.load(Ordering::SeqCst), 1);
// replay with original scoped key/partial input: same Task, no fingerprint conflict
let replay = app
.accept_background(
&caller,
TOOL_NAME,
None,
"input-key-1",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("replay accepted");
assert_eq!(replay.task_id, task_id);
server.abort();
}
pub(crate) async fn start_and_suspend(
registry: &PostgresRegistry,
app: &TaskApplication,
caller: &crank_community_mcp::TaskPrincipal,
scoped_key: &str,
) -> (TaskId, String, u32) {
let acceptance = app
.accept_background(
caller,
TOOL_NAME,
None,
scoped_key,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("acceptance");
let (stop, worker_task) = spawn_worker(registry, app);
assert!(
wait_for_state(
registry,
acceptance.task_id,
"input_required",
Duration::from_secs(10)
)
.await,
"worker must suspend for missing email"
);
stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
let (wait_id, expected_version) = wait_row(registry, acceptance.task_id).await;
(acceptance.task_id, wait_id, expected_version)
}
async fn wait_row(registry: &PostgresRegistry, task_id: TaskId) -> (String, u32) {
let row =
sqlx::query("select wait_id, expected_version from task_input_requests where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
(
row.get::<String, _>("wait_id"),
u32::try_from(row.get::<i32, _>("expected_version")).unwrap_or_default(),
)
}
pub(crate) async fn audit_events(
registry: &PostgresRegistry,
task_id: TaskId,
) -> Vec<(String, Option<String>)> {
sqlx::query("select event, reason_code from task_input_audit where task_id = $1 order by id")
.bind(task_id.as_uuid())
.fetch_all(registry.pool())
.await
.unwrap()
.into_iter()
.map(|row| (row.get("event"), row.get("reason_code")))
.collect()
}
pub(crate) async fn admission_entries(
registry: &PostgresRegistry,
task_id: TaskId,
) -> Vec<(String, i64)> {
sqlx::query(
"select state, count(*)::bigint as count from execution_admission_entries
where task_id = $1 group by state order by state",
)
.bind(task_id.as_uuid())
.fetch_all(registry.pool())
.await
.unwrap()
.into_iter()
.map(|row| (row.get("state"), row.get("count")))
.collect()
}
fn key_id(principal: &crank_community_mcp::TaskPrincipal) -> crank_core::PlatformApiKeyId {
match principal {
crank_community_mcp::TaskPrincipal::Agent { credential, .. } => credential
.platform_api_key_id
.clone()
.expect("agent principal carries a key id"),
crank_community_mcp::TaskPrincipal::Admin { .. } => panic!("agent principal expected"),
}
}
#[tokio::test]
async fn invalid_schema_foreign_scope_and_revoked_key_never_transition_or_dispatch() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "input-scope").await;
let app = application(&registry).await;
let (task_id, wait_id, expected_version) =
start_and_suspend(&registry, &app, &caller, "input-key-scope").await;
// Invalid answer: server-side schema validation refuses it, nothing transitions and nothing leaks.
let invalid = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-invalid",
&json!({}),
)
.await
.expect_err("the pinned schema requires email");
assert!(
matches!(
invalid,
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::Invalid { .. })
),
"unexpected error: {invalid:?}"
);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("input_required")
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert!(
audit_events(&registry, task_id)
.await
.iter()
.all(|(event, _)| event == "input_required"),
"a rejected answer must not be audited as accepted"
);
let stored_snapshot: serde_json::Value =
sqlx::query_scalar("select schema_snapshot from task_input_requests where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert!(!stored_snapshot.to_string().contains("user@example.com"));
// A different key of the same agent carries a different authorization scope: opaque not-found.
let foreign = principal(&registry, "input-foreign").await;
let foreign_result = app
.update_input(
&foreign,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-foreign",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("a foreign scope cannot continue the Task");
assert_eq!(
foreign_result,
crank_community_mcp::TaskApplicationError::NotFound
);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("input_required")
);
// Positive control: the accepting principal (same key/scope) completes the transition.
let accepted = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-valid",
&json!({"email": "user@example.com"}),
)
.await
.expect("valid answer for the accepting scope");
assert_eq!(accepted.state, TaskState::Working);
assert!(!accepted.replayed);
// Finish the positive-control Task through a real worker so the revocation case starts from a
// known dispatch count instead of leaving resumable work behind.
let (finish_stop, finish_worker) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_id, "completed", Duration::from_secs(10)).await,
"the positive-control Task completes"
);
finish_stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), finish_worker).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
// Revocation race: the accepting key is revoked after suspension. The same waiting scope must
// fail closed inside the update transaction and never revive or dispatch the Task.
let revocable = principal(&registry, "input-revoked").await;
let (revoked_task, revoked_wait, revoked_version) =
start_and_suspend(&registry, &app, &revocable, "input-key-revoked").await;
registry
.revoke_platform_api_key(
&common::test_workspace_id(),
&key_id(&revocable),
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
let revoked_result = app
.update_input(
&revocable,
TOOL_NAME,
revoked_task,
&revoked_wait,
revoked_version,
"update-revoked",
&json!({"email": "revoked@example.com"}),
)
.await
.expect_err("a revoked key cannot continue the Task");
assert_eq!(
revoked_result,
crank_community_mcp::TaskApplicationError::NotFound
);
assert_eq!(
state_of(&registry, revoked_task).await.as_deref(),
Some("input_required")
);
assert_eq!(
calls.load(Ordering::SeqCst),
1,
"the revoked answer must not add an outbound call"
);
server.abort();
}
#[tokio::test]
async fn replay_conflict_and_stale_answers_have_one_typed_outcome() {
let registry = common::test_registry().await;
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, "http://127.0.0.1:9", policy).await;
let caller = principal(&registry, "input-replay").await;
let app = application(&registry).await;
let (task_id, wait_id, expected_version) =
start_and_suspend(&registry, &app, &caller, "input-key-replay").await;
let first = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-key",
&json!({"email": "user@example.com"}),
)
.await
.expect("first valid answer");
assert!(!first.replayed);
assert_eq!(first.state, TaskState::Working);
// Identical replay returns the stored outcome without a second CAS/audit.
let replay = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-key",
&json!({"email": "user@example.com"}),
)
.await
.expect("identical replay");
assert!(replay.replayed);
assert_eq!(replay.version, first.version);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("working")
);
let accepted_audit = audit_events(&registry, task_id)
.await
.into_iter()
.filter(|(event, _)| event == "input_accepted")
.count();
assert_eq!(accepted_audit, 1);
// Same key with another payload is a fingerprint conflict.
let conflict = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-key",
&json!({"email": "other@example.com"}),
)
.await
.expect_err("same key with another payload conflicts");
assert_eq!(
conflict,
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::FingerprintConflict)
);
// Another key after a committed answer, and a foreign wait id, are state conflicts.
let other_key = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
first.version,
"update-key-2",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("another key after commit conflicts");
assert_eq!(
other_key,
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
);
let foreign_wait = app
.update_input(
&caller,
TOOL_NAME,
task_id,
"wait-does-not-exist",
first.version,
"update-key-3",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("a foreign wait id conflicts");
assert_eq!(
foreign_wait,
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
);
// A stale generation on a fresh wait is refused; the valid generation still works.
let (stale_task, stale_wait, stale_version) =
start_and_suspend(&registry, &app, &caller, "input-key-stale").await;
let stale = app
.update_input(
&caller,
TOOL_NAME,
stale_task,
&stale_wait,
stale_version.saturating_sub(1).max(1),
"update-stale",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("a stale generation cannot be continued");
assert_eq!(
stale,
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
);
let repaired = app
.update_input(
&caller,
TOOL_NAME,
stale_task,
&stale_wait,
stale_version,
"update-stale",
&json!({"email": "user@example.com"}),
)
.await
.expect("the current generation still accepts the answer");
assert!(!repaired.replayed);
}
#[tokio::test]
async fn ttl_expiry_fails_with_audit_and_releases_capacity_without_dispatch() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
// The wait TTL is the remaining pinned absolute execution deadline.
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 1_200,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "input-ttl").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"input-key-ttl",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("acceptance");
let task_id = acceptance.task_id;
let (stop, worker_task) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_id, "input_required", Duration::from_secs(5)).await,
"worker must suspend"
);
assert!(
wait_for_state(&registry, task_id, "failed", Duration::from_secs(6)).await,
"the per-poll sweeper must fail the expired wait"
);
stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
let (wait_id, expected_version) = wait_row(&registry, task_id).await;
let events = audit_events(&registry, task_id).await;
assert!(
events.iter().any(|(event, reason)| event == "input_expired"
&& reason.as_deref() == Some("input_ttl_expired")),
"expiry reason must be audited: {events:?}"
);
assert_eq!(admission_entries(&registry, task_id).await, Vec::new());
assert_eq!(calls.load(Ordering::SeqCst), 0);
// A late answer cannot revive the failed Task.
let late = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-late",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("an expired wait never revives");
assert!(
matches!(
late,
crank_community_mcp::TaskApplicationError::Task(
crank_core::TaskError::InputExpired | crank_core::TaskError::StateConflict
)
),
"unexpected late answer error: {late:?}"
);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("failed")
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
server.abort();
}
#[tokio::test]
async fn competing_identical_updates_commit_once_and_keep_one_permit() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "input-race").await;
let app = application(&registry).await;
let (task_id, wait_id, expected_version) =
start_and_suspend(&registry, &app, &caller, "input-key-race").await;
assert_eq!(
admission_entries(&registry, task_id).await,
vec![("executing".to_owned(), 1)],
"the waiting Task keeps exactly one executing permit"
);
let payload = json!({"email": "user@example.com"});
let first = app.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-race",
&payload,
);
let second = app.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-race",
&payload,
);
let (first, second) = tokio::join!(first, second);
let mut outcomes = [
first.expect("competing answer"),
second.expect("competing answer"),
];
outcomes.sort_by_key(|outcome| outcome.replayed);
assert!(!outcomes[0].replayed);
assert!(outcomes[1].replayed);
assert_eq!(outcomes[0].version, outcomes[1].version);
assert_eq!(
audit_events(&registry, task_id)
.await
.into_iter()
.filter(|(event, _)| event == "input_accepted")
.count(),
1
);
assert_eq!(
admission_entries(&registry, task_id).await,
vec![("executing".to_owned(), 1)],
"the update must not grant a second permit"
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
let (stop, worker_task) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_id, "completed", Duration::from_secs(10)).await,
"the resumed Task completes"
);
stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(admission_entries(&registry, task_id).await, Vec::new());
server.abort();
}
@@ -0,0 +1,457 @@
//! ATDD Story 4.5 QA1 guards (F1/F3): one pinned deadline for several waiting Tasks, cancellation
//! while the Task waits for input and refusal of a stale lease/fence around the continuation seam.
use super::common;
use super::input_continuation::{
TOOL_NAME, admission_entries, application, audit_events, principal, seed_tool, spawn_worker,
start_and_suspend, state_of, wait_for_state,
};
use super::unsampled_correlation;
use axum::Router;
use axum::http::StatusCode;
use axum::routing::post;
use crank_core::{
BeginAttempt, NoTasksFallback, TASK_INPUT_SCHEMA_FORMAT_VERSION, TaskInputRequest,
TaskInputStore, TaskState, TaskStore, ToolExecutionMode, ToolExecutionPolicy,
};
use crank_registry::{AdmissionLimits, PostgresTaskStore};
use serde_json::json;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use time::OffsetDateTime;
#[tokio::test]
async fn two_waits_with_one_pinned_deadline_and_cancelled_leftover_do_not_block_suspension() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "deadline-cancel").await;
let app = application(&registry).await;
let acc_a = app
.accept_background(
&caller,
TOOL_NAME,
None,
"deadline-key-a",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("accept a");
let acc_b = app
.accept_background(
&caller,
TOOL_NAME,
None,
"deadline-key-b",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("accept b");
let task_a = acc_a.task_id;
let task_b = acc_b.task_id;
let pinned_ms = (OffsetDateTime::now_utc().unix_timestamp() as f64 * 1000.0) + 60_000.0;
for tid in [task_a, task_b] {
sqlx::query("update tasks set execution_deadline = to_timestamp($1::double precision/1000.0) where task_id = $2")
.bind(pinned_ms)
.bind(tid.as_uuid())
.execute(registry.pool())
.await
.unwrap();
}
let (stop, worker_task) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_a, "input_required", Duration::from_secs(10)).await,
"a must suspend"
);
assert!(
wait_for_state(&registry, task_b, "input_required", Duration::from_secs(10)).await,
"b must suspend"
);
stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(calls.load(Ordering::SeqCst), 0);
// Both active waits carry the same absolute TTL value: the per-Task uniqueness above is the only
// uniqueness, so this must not be a UNIQUE index on expires_at.
let same_deadline_wait: i64 = sqlx::query_scalar(
"select count(*)::bigint from task_input_requests a
join task_input_requests b on b.task_id = $2
where a.task_id = $1 and a.accepted_at is null and b.accepted_at is null
and a.expires_at = b.expires_at",
)
.bind(task_a.as_uuid())
.bind(task_b.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(
same_deadline_wait, 1,
"both waits share the pinned deadline"
);
let outcome = app
.cancel(&caller, TOOL_NAME, task_a, None)
.await
.expect("cancel a");
assert!(matches!(
outcome,
crank_core::TaskCancelOutcome::Cancelled(_)
));
assert_eq!(
state_of(&registry, task_a).await.as_deref(),
Some("cancelled")
);
assert_eq!(admission_entries(&registry, task_a).await, Vec::new());
// accept C with same pinned deadline
let acc_c = app
.accept_background(
&caller,
TOOL_NAME,
None,
"deadline-key-c",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("accept c");
let task_c = acc_c.task_id;
sqlx::query("update tasks set execution_deadline = to_timestamp($1::double precision/1000.0) where task_id = $2")
.bind(pinned_ms)
.bind(task_c.as_uuid())
.execute(registry.pool())
.await
.unwrap();
let (stop2, worker_task2) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_c, "input_required", Duration::from_secs(10)).await,
"c must suspend despite leftover a"
);
stop2.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task2).await;
assert_eq!(
state_of(&registry, task_b).await.as_deref(),
Some("input_required")
);
let c_deadline_ms: i64 = sqlx::query_scalar(
"select (extract(epoch from expires_at) * 1000)::bigint
from task_input_requests where task_id = $1",
)
.bind(task_c.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(
c_deadline_ms, pinned_ms as i64,
"the third wait reuses the same pinned deadline value"
);
let a_accepted: Option<OffsetDateTime> =
sqlx::query_scalar("select accepted_at from task_input_requests where task_id = $1")
.bind(task_a.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert!(
a_accepted.is_none(),
"cancellation leaves the wait row without an accepted receipt"
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
server.abort();
}
#[tokio::test]
async fn cancel_while_input_required_never_revives_or_dispatches() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "cancel-wait").await;
let app = application(&registry).await;
let (task_id, wait_id, expected_version) =
start_and_suspend(&registry, &app, &caller, "input-key-cancel").await;
assert_eq!(
admission_entries(&registry, task_id).await,
vec![("executing".to_owned(), 1)]
);
let outcome = app
.cancel(&caller, TOOL_NAME, task_id, None)
.await
.expect("cancel");
assert!(matches!(
outcome,
crank_core::TaskCancelOutcome::Cancelled(_)
));
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("cancelled")
);
assert_eq!(admission_entries(&registry, task_id).await, Vec::new());
let late = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-late",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("late answer must not revive");
assert!(
matches!(
late,
crank_community_mcp::TaskApplicationError::Task(
crank_core::TaskError::StateConflict | crank_core::TaskError::InputExpired
)
),
"unexpected: {late:?}"
);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("cancelled")
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
let events = audit_events(&registry, task_id).await;
assert!(events.iter().any(|(e, _)| e == "input_required"));
assert!(!events.iter().any(|(e, _)| e == "input_accepted"));
let attempts = app
.attempts(&caller, TOOL_NAME, task_id)
.await
.expect("attempts");
assert!(attempts.is_empty());
server.abort();
}
#[tokio::test]
async fn stale_lease_fence_cannot_suspend_or_dispatch() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "stale-fence").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"stale-key",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let task_id = acceptance.task_id;
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let mut old_claims = store
.claim_batch("worker-old", 1, 50)
.await
.expect("old claim");
assert!(!old_claims.is_empty());
let old = old_claims.remove(0);
tokio::time::sleep(Duration::from_millis(120)).await;
let mut new_claims = store
.claim_batch("worker-new", 1, 10_000)
.await
.expect("new claim");
assert!(!new_claims.is_empty());
let new = new_claims.remove(0);
assert!(new.fence > old.fence);
assert_eq!(new.record.task_id, task_id);
let request = TaskInputRequest {
wait_id: "stale-wait".to_string(),
expected_version: old.record.version + 1,
schema_revision: 1,
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
schema_snapshot: json!({}),
projected_schema: json!({}),
prompt_code: "required_input_missing".to_string(),
missing_fields: vec!["$.email".to_string()],
expires_at_unix_ms: OffsetDateTime::now_utc().unix_timestamp() * 1000 + 30_000,
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
};
let suspended =
<PostgresTaskStore as TaskInputStore>::suspend_for_input(&*store, &old, request).await;
assert!(
suspended.is_err(),
"stale fence must not suspend: {suspended:?}"
);
let attempt_id = "attempt-stale".to_string();
let checkpoint = store
.checkpoint_dispatch(BeginAttempt {
task_id,
owner: old.owner.clone(),
fence: old.fence,
expected_version: old.record.version,
attempt_id,
input_fingerprint: old.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await;
assert!(
checkpoint.is_err(),
"stale fence must not checkpoint: {checkpoint:?}"
);
assert_ne!(
state_of(&registry, task_id).await.as_deref(),
Some("input_required")
);
let attempt_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempt_count, 0);
let req_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_input_requests where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(req_count, 0);
assert_eq!(calls.load(Ordering::SeqCst), 0);
// Positive control: the same request under the current lease suspends the Task exactly once, so
// the refusal above is the stale fence and not an invalid request shape.
let request = TaskInputRequest {
wait_id: "current-wait".to_string(),
expected_version: new.record.version + 1,
schema_revision: 1,
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
schema_snapshot: json!({}),
projected_schema: json!({}),
prompt_code: "required_input_missing".to_string(),
missing_fields: vec!["$.email".to_string()],
expires_at_unix_ms: new.record.pins.execution_deadline_unix_ms,
};
let suspended =
<PostgresTaskStore as TaskInputStore>::suspend_for_input(&*store, &new, request)
.await
.expect("the current lease suspends");
assert_eq!(suspended.state, TaskState::InputRequired);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("input_required")
);
assert_eq!(
admission_entries(&registry, task_id).await,
vec![("executing".to_owned(), 1)]
);
// The stale fence still cannot checkpoint a dispatch of this generation.
let checkpoint = store
.checkpoint_dispatch(BeginAttempt {
task_id,
owner: old.owner.clone(),
fence: old.fence,
expected_version: old.record.version,
attempt_id: "attempt-stale-after-suspend".to_string(),
input_fingerprint: old.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await;
assert!(
checkpoint.is_err(),
"stale fence must not checkpoint: {checkpoint:?}"
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
server.abort();
}
@@ -14,13 +14,14 @@ use std::time::Duration;
use crank_community_mcp::{TaskApplication, TaskApplicationError, TaskPrincipal};
use crank_core::{
AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, TaskState,
ToolExecutionMode, ToolExecutionPolicy,
AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, RequestId,
TaskAcceptRequest, TaskEnvelope, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore,
TaskSupportIds, ToolExecutionMode, ToolExecutionPolicy,
};
use crank_registry::{
AdmissionLimits, CreateVersionRequest, PostgresRegistry, PostgresTaskStore, PublishRequest,
};
use crank_runtime::{InMemoryCoordinationStateStore, TaskService};
use crank_runtime::{InMemoryCoordinationStateStore, SecretCrypto, TaskService};
use serde_json::json;
use sqlx::{PgPool, Row};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
@@ -149,7 +150,9 @@ async fn publish_tool_version(registry: &PostgresRegistry, version: u32) {
.unwrap();
}
fn task_service(pool: PgPool) -> TaskService {
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
fn task_service(pool: PgPool, crypto: SecretCrypto) -> TaskService {
let store = PostgresTaskStore::new(
pool,
AdmissionLimits {
@@ -161,15 +164,27 @@ fn task_service(pool: PgPool) -> TaskService {
Default::default(),
)
.expect("valid task store configuration");
TaskService::new(std::sync::Arc::new(store))
TaskService::with_secret_crypto(std::sync::Arc::new(store), std::sync::Arc::new(crypto))
}
fn application(registry: &PostgresRegistry) -> TaskApplication {
/// The application entry under the same encrypted-input authority the production process uses.
async fn application(registry: &PostgresRegistry) -> TaskApplication {
let crypto = SecretCrypto::new(TEST_MASTER_KEY).expect("test master key");
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("test master-key identity registered");
TaskApplication::new(
registry.clone(),
Duration::from_millis(0),
std::sync::Arc::new(InMemoryCoordinationStateStore::default()),
task_service(registry.pool().clone()),
task_service(registry.pool().clone(), crypto),
)
}
@@ -243,7 +258,7 @@ async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() {
)
.await;
let correlation = unsampled_correlation();
let app = application(&registry);
let app = application(&registry).await;
let acceptance = app
.accept_background(
@@ -273,7 +288,7 @@ async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() {
assert_eq!(counts(registry.pool()).await, (1, 1));
// A freshly constructed application + service reads the same durable identity and pins.
let reader = application(&registry);
let reader = application(&registry).await;
let record = reader
.status(&caller, TOOL_NAME, acceptance.task_id)
.await
@@ -317,7 +332,7 @@ async fn a_new_publication_never_rebinds_an_accepted_task() {
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = application(&registry);
let app = application(&registry).await;
let accepted = app
.accept_background(
&caller,
@@ -397,7 +412,7 @@ async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = application(&registry);
let app = application(&registry).await;
let accepted = app
.accept_background(
&owner,
@@ -433,7 +448,7 @@ async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
));
// A synchronous publication is not accepted as background work at all.
let synchronous_app = application(&registry);
let synchronous_app = application(&registry).await;
let refused = synchronous_app
.accept_background(
&owner,
@@ -453,7 +468,7 @@ async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
async fn admin_owned_tasks_are_scoped_to_that_admin_and_workspace() {
let registry = common::test_registry().await;
seed_tool(&registry, background_policy()).await;
let app = application(&registry);
let app = application(&registry).await;
let admin = TaskPrincipal::Admin {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
@@ -549,7 +564,7 @@ async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() {
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = application(&registry);
let app = application(&registry).await;
assert!(app.health().await.is_ok());
// Fail closed: without the Task schema the entry accepts nothing and reports dependency failure.
@@ -589,3 +604,56 @@ async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() {
assert_eq!(body["checks"]["durable_tasks"], json!("not_ready"));
server.abort();
}
/// Commits and resolves exactly one durable attempt of the current lease through the real store.
///
/// Story 4.4 moved the dispatch checkpoint and the attempt resolution into the runtime seam, so a
/// worker-level test double persists exactly one attempt instead of inventing a second retry loop.
pub(crate) async fn commit_attempt(
context: &crank_runtime::TaskAttemptContext,
outcome: crank_core::AttemptOutcome,
) -> crank_runtime::TaskAttemptOutcome {
use crank_core::{AttemptResolution, BeginAttempt, TaskState};
let lease = context.lease();
let begin = BeginAttempt {
task_id: lease.record.task_id,
owner: lease.owner.clone(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: format!("attempt-{}", uuid::Uuid::now_v7()),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: lease.record.pins.retry.max_attempts > 1,
};
match context.store().checkpoint_dispatch(begin).await {
Ok(attempt) => match context.store().resolve_attempt(&attempt, outcome).await {
Ok(AttemptResolution::Finished(task)) if task.state == TaskState::Completed => {
crank_runtime::TaskAttemptOutcome::Completed
}
Ok(AttemptResolution::Finished(_)) => crank_runtime::TaskAttemptOutcome::Failed {
code: "test_failure".to_owned(),
},
Ok(AttemptResolution::Unknown(_)) => crank_runtime::TaskAttemptOutcome::OutcomeUnknown,
Ok(AttemptResolution::RetryAt { .. }) | Err(_) => {
crank_runtime::TaskAttemptOutcome::Stopped
}
},
Err(_) => crank_runtime::TaskAttemptOutcome::Stopped,
}
}
/// Bounded success outcome used by the worker-level test doubles.
pub(crate) fn completed_outcome(value: serde_json::Value) -> crank_core::AttemptOutcome {
crank_core::AttemptOutcome::Completed(
crank_core::TaskEnvelope::try_redacted(&value).expect("test result is bounded"),
)
}
mod attempt_guards;
mod attempt_trace;
mod cancel_chain;
mod input_continuation;
mod input_continuation_guards;
mod shutdown_window;
mod task_worker;
@@ -0,0 +1,220 @@
//! Story 4.3 QA caveat proof: the narrow post-checkpoint / pre-permit stop window.
//!
//! Split from `task_worker.rs` to keep both integration files inside the code-health limit.
use super::*;
/// A stop that lands after the durable dispatch checkpoint but before the sealed dispatch permit
/// exists finalizes the Task as failed(`shutdown_before_dispatch`) without any outbound call, and
/// does not consume work that is still queued.
///
/// The window is made deterministic instead of raced: a `BEFORE UPDATE` trigger delays only the
/// durable dispatch checkpoint, so the stop flag is set while that checkpoint statement is still in
/// flight (before `TaskDispatchPermit`/`TaskHandler::execute` can exist).
#[tokio::test]
async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream() {
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let registry = common::test_registry().await;
// The test registry exposes a single pooled connection: delaying the checkpoint inside its own
// statement keeps the window deterministic without competing for a second connection.
sqlx::raw_sql(sqlx::AssertSqlSafe(
"create function delay_task_dispatch_checkpoint() returns trigger language plpgsql as $$
begin
if new.current_attempt_id is not null and old.current_attempt_id is null then
perform pg_sleep(1.5);
end if;
return new;
end $$;
create trigger delay_task_dispatch_checkpoint before update on tasks
for each row execute function delay_task_dispatch_checkpoint();",
))
.execute(registry.pool())
.await
.expect("checkpoint delay trigger");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let mut accepted = Vec::new();
for (seed, scoped_key) in [("q", "a"), ("r", "c")] {
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_stop_window".to_owned(),
platform_api_key_id: None,
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 1,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: scoped_key.repeat(64),
input_fingerprint: scoped_key.repeat(64),
input: TaskEnvelope::try_redacted(&json!({ "report": seed })).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&crank_core::TraceContext::parse(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
)
.unwrap(),
),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
};
accepted.push(store.accept(request).await.expect("PG accepts"));
}
// The first revalidation waits for the test; the second Task is never claimed (batch_limit = 1).
let revalidate_entered = Arc::new(tokio::sync::Semaphore::new(0));
let revalidate_release = Arc::new(tokio::sync::Semaphore::new(0));
let executed = Arc::new(AtomicUsize::new(0));
struct GatedHandler {
entered: Arc<tokio::sync::Semaphore>,
release: Arc<tokio::sync::Semaphore>,
executed: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl TaskHandler for GatedHandler {
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
self.entered.add_permits(1);
let _ = self.release.acquire().await;
Ok(())
}
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
// The trigger delays the durable checkpoint; a stop that lands while it is in flight must
// finalize failed(shutdown_before_dispatch) without an outbound send.
let begin = crank_core::BeginAttempt {
task_id: context.lease().record.task_id,
owner: context.lease().owner.clone(),
fence: context.lease().fence,
expected_version: context.lease().record.version,
attempt_id: "attempt-stop-window".to_owned(),
input_fingerprint: context.lease().record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
};
let Ok(attempt) = context.store().checkpoint_dispatch(begin).await else {
return TaskAttemptOutcome::Stopped;
};
if context.is_stopping() {
let _ = context
.store()
.resolve_attempt(
&attempt,
crank_core::AttemptOutcome::Failed {
error: crank_core::TaskEnvelope::try_redacted(&json!({
"code": "shutdown_before_dispatch",
"phase": "before_dispatch",
}))
.expect("bounded error envelope"),
retryable: false,
phase: crank_core::AttemptPhase::BeforeDispatch,
},
)
.await;
return TaskAttemptOutcome::Failed {
code: "shutdown_before_dispatch".to_owned(),
};
}
self.executed.fetch_add(1, Ordering::SeqCst);
commit_attempt(&context, completed_outcome(json!({ "ok": true }))).await
}
}
let handler: Arc<dyn TaskHandler> = Arc::new(GatedHandler {
entered: revalidate_entered.clone(),
release: revalidate_release.clone(),
executed: executed.clone(),
});
let config = TaskWorkerConfig {
owner: "runtime-worker-stop-window".to_owned(),
batch_limit: 1,
lease_ms: 30_000,
poll_interval: Duration::from_millis(50),
renewal_interval: Duration::from_secs(20),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(20),
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
tokio::time::timeout(Duration::from_secs(5), revalidate_entered.acquire())
.await
.expect("worker must claim the first Task and enter revalidation")
.expect("revalidation gate is open")
.forget();
// Let revalidation continue; the checkpoint now runs while the trigger holds it for 1.5s.
revalidate_release.add_permits(1);
tokio::time::sleep(Duration::from_millis(300)).await;
// The stop now lands while the durable checkpoint statement is still in flight: the worker may
// finalize the Task, but no permit exists yet, so no outbound call can start.
let stop_requested = std::time::Instant::now();
stop_flag.store(true, Ordering::SeqCst);
tokio::time::timeout(Duration::from_secs(30), worker_task)
.await
.expect("worker stops inside the 30s budget")
.expect("worker exits cleanly")
.expect("worker reports no error");
assert!(
stop_requested.elapsed() >= Duration::from_millis(500),
"the checkpoint must have still been in flight when the stop arrived"
);
assert_eq!(
executed.load(Ordering::SeqCst),
0,
"a stop between checkpoint and permit must never reach the upstream adapter"
);
let access = TaskAccess {
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_stop_window".to_owned(),
};
let record = store.read(&access, accepted[0].task_id).await.unwrap();
assert_eq!(record.state, TaskState::Failed);
assert_eq!(record.attempts, 1, "the durable checkpoint did commit");
assert!(record.current_attempt_id.is_some());
assert_eq!(
record
.error
.as_ref()
.and_then(|error| error.value().get("code"))
.and_then(|code| code.as_str()),
Some("shutdown_before_dispatch")
);
// Work that was still queued is not consumed by the stopped worker.
let queued = store.read(&access, accepted[1].task_id).await.unwrap();
assert_eq!(queued.state, TaskState::Queued);
let recovered = store
.claim_batch("worker_after_stop", 4, 10_000)
.await
.expect("queued work stays claimable");
assert_eq!(recovered.len(), 1);
assert_eq!(recovered[0].record.task_id, accepted[1].task_id);
}
@@ -0,0 +1,821 @@
//! Story 4.3 ATDD: fenced durable Task worker leases, recovery and real upstream dispatch.
//!
//! Split from `durable_tasks.rs` to keep both integration files inside the code-health limit.
use super::*;
/// Store-level positive control only: PostgreSQL leases/fencing/recovery work without Valkey or a
/// task-worker process. This does not exercise TaskWorker startup, a bounded authoritative scan,
/// graceful shutdown, outbound dispatch, or runtime recovery timing, which remain unimplemented.
#[tokio::test]
async fn task_worker_store_lease_recovery_contract() {
let registry = common::test_registry().await;
let store = PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration");
// Positive control: no Valkey or worker process is involved; PostgreSQL alone accepts and
// leases durable work, preserving the same admission-backed intent.
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
platform_api_key_id: None,
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 1,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: "a".repeat(64),
input_fingerprint: "b".repeat(64),
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&crank_core::TraceContext::parse(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
)
.unwrap(),
),
// This store-level control exercises lease/fence behavior, not the encrypted acceptance path.
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
};
let accepted = store.accept(request).await.expect("PG accepts");
let leases = store
.claim_batch("worker_1", 4, 10_000)
.await
.expect("claim");
assert_eq!(leases.len(), 1);
assert_eq!(leases[0].record.task_id, accepted.task_id);
let lease = leases[0].clone();
// A stale or competing owner cannot win CAS with the issued fencing token.
assert!(
store
.renew_lease("worker_2", lease.record.task_id, lease.fence, 10_000)
.await
.is_err()
);
assert!(
store
.write_progress(
"worker_1",
lease.record.task_id,
lease.fence + 1,
TaskEnvelope::try_redacted(&json!({"progress": 1})).unwrap(),
)
.await
.is_err()
);
// A stale fencing token can never checkpoint dispatch, even under the issuing owner.
assert!(
store
.checkpoint_dispatch(crank_core::BeginAttempt {
task_id: lease.record.task_id,
owner: "worker_1".to_owned(),
fence: lease.fence + 1,
expected_version: lease.record.version,
attempt_id: "attempt-stale".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await
.is_err()
);
// The valid owner still commits the dispatch checkpoint before any outbound work.
let committed = store
.checkpoint_dispatch(crank_core::BeginAttempt {
task_id: lease.record.task_id,
owner: "worker_1".to_owned(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: "attempt-1".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await
.unwrap();
assert_eq!(committed.attempt.attempt_id, "attempt-1");
assert_eq!(
committed.task.current_attempt_id.as_deref(),
Some("attempt-1")
);
}
#[tokio::test]
async fn task_worker_runtime_dispatch_and_recovery_contract() {
use async_trait::async_trait;
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let registry = common::test_registry().await;
let pool = registry.pool().clone();
let store = Arc::new(
PostgresTaskStore::new(
pool.clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let observed = Arc::new(AtomicUsize::new(0));
// Positive control: PostgreSQL alone accepts the durable Task.
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_runtime".to_owned(),
platform_api_key_id: None,
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 1,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: "r".repeat(64),
input_fingerprint: "b".repeat(64),
input: TaskEnvelope::try_redacted(&json!({"email": "q3"})).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&crank_core::TraceContext::parse(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
)
.unwrap(),
),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
};
let accepted = store.accept(request).await.expect("PG accepts");
let task_id = accepted.task_id;
struct RuntimeHandler {
observed: Arc<AtomicUsize>,
}
#[async_trait]
impl TaskHandler for RuntimeHandler {
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
Ok(())
}
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
self.observed.fetch_add(1, Ordering::SeqCst);
commit_attempt(&context, completed_outcome(serde_json::json!({"ok": true}))).await
}
}
let handler: Arc<dyn TaskHandler> = Arc::new(RuntimeHandler {
observed: observed.clone(),
});
// PG-only authoritative dispatch: no Valkey or upstream process is involved.
let config = TaskWorkerConfig {
owner: "runtime-worker-1".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(100),
renewal_interval: Duration::from_millis(1_000),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let access = TaskAccess {
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_runtime".to_owned(),
};
let mut state = None;
for _ in 0..100 {
tokio::time::sleep(Duration::from_millis(100)).await;
if let Ok(record) = store.read(&access, task_id).await {
state = Some(record.state);
if record.state == TaskState::Completed {
break;
}
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(state, Some(TaskState::Completed));
assert_eq!(
observed.load(Ordering::SeqCst),
1,
"exactly one outbound attempt"
);
// Terminal state is not duplicated.
let record = store.read(&access, task_id).await.expect("terminal read");
assert_eq!(record.state, TaskState::Completed);
// Bounded shutdown with no fresh work returns promptly.
let idle_config = TaskWorkerConfig {
owner: "runtime-worker-idle".to_owned(),
poll_interval: Duration::from_millis(50),
shutdown_timeout: Duration::from_secs(2),
..TaskWorkerConfig::default()
};
let idle_worker = TaskWorker::new(
store.clone(),
handler_for_no_work(store.clone()),
idle_config,
)
.expect("valid idle worker");
idle_worker.stop();
let started = std::time::Instant::now();
idle_worker.run().await.expect("idle worker stops");
assert!(started.elapsed() <= Duration::from_secs(30));
}
/// Story 4.3 ATDD: failed pre-dispatch revalidation finishes the attempt without any outbound call.
#[tokio::test]
async fn task_worker_revalidation_denial_never_dispatches() {
use crank_core::{TaskAccess, TaskError, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let registry = common::test_registry().await;
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_denied".to_owned(),
platform_api_key_id: None,
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 1,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: "d".repeat(64),
input_fingerprint: "b".repeat(64),
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&crank_core::TraceContext::parse(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
)
.unwrap(),
),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
};
let accepted = store.accept(request).await.expect("PG accepts");
struct DeniedHandler {
observed: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl TaskHandler for DeniedHandler {
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
Err(TaskWorkerError::Store(TaskError::invalid(
"publication or credential revoked",
)))
}
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
self.observed.fetch_add(1, Ordering::SeqCst);
commit_attempt(&context, completed_outcome(json!({"ok": true}))).await
}
}
let observed = Arc::new(AtomicUsize::new(0));
let handler: Arc<dyn TaskHandler> = Arc::new(DeniedHandler {
observed: observed.clone(),
});
let config = TaskWorkerConfig {
owner: "runtime-worker-denied".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let access = TaskAccess {
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_denied".to_owned(),
};
let mut state = None;
for _ in 0..100 {
tokio::time::sleep(Duration::from_millis(50)).await;
if let Ok(record) = store.read(&access, accepted.task_id).await {
state = Some(record.state);
if record.state == TaskState::Failed {
break;
}
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(state, Some(TaskState::Failed));
assert_eq!(
observed.load(Ordering::SeqCst),
0,
"invalidated work never reaches the upstream"
);
}
fn handler_for_no_work(
_store: std::sync::Arc<PostgresTaskStore>,
) -> std::sync::Arc<dyn crank_runtime::TaskHandler> {
struct NoWorkHandler;
#[async_trait::async_trait]
impl crank_runtime::TaskHandler for NoWorkHandler {
async fn revalidate(
&self,
_lease: &crank_core::TaskLease,
) -> Result<(), crank_runtime::TaskWorkerError> {
Ok(())
}
async fn execute(
&self,
context: crank_runtime::TaskAttemptContext,
) -> crank_runtime::TaskAttemptOutcome {
commit_attempt(&context, completed_outcome(serde_json::json!({"ok": true}))).await
}
}
std::sync::Arc::new(NoWorkHandler)
}
/// Story 4.3 end-to-end ATDD: the production durable-Task handler, the real runtime executor and a
/// real local HTTP upstream observe exactly one dispatch after the fenced checkpoint, while the
/// accepted parent trace is continued on the wire.
#[tokio::test]
async fn task_worker_dispatches_one_real_upstream_attempt_with_parent_trace() {
use axum::Router;
use axum::http::{HeaderMap, StatusCode};
use axum::routing::post;
use crank_community_mcp::TaskWorkerHandler;
use crank_core::{AgentOperationBinding, PlatformApiKeyScope};
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let seen_traceparent = Arc::new(Mutex::new(None::<String>));
let app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
let seen = Arc::clone(&seen_traceparent);
move |headers: HeaderMap| {
let calls = Arc::clone(&calls);
let seen = Arc::clone(&seen);
async move {
calls.fetch_add(1, Ordering::SeqCst);
if let Some(value) = headers
.get("traceparent")
.and_then(|value| value.to_str().ok())
{
*seen.lock().unwrap() = Some(value.to_owned());
}
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(background_policy());
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(&registry, &operation.id, 1).await;
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(
&registry,
"task-worker-e2e",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let correlation = unsampled_correlation();
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("test master-key identity registered");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let tasks = TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
);
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
"worker-e2e-key",
&json!({"email": "user@example.com"}),
&correlation,
)
.await
.expect("durable acceptance with executable input");
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build();
let handler = Arc::new(TaskWorkerHandler::from_application(
&tasks,
registry.clone(),
runtime,
crypto,
));
let config = TaskWorkerConfig {
owner: "runtime-worker-e2e".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let mut state = None;
for _ in 0..200 {
tokio::time::sleep(Duration::from_millis(50)).await;
let current: Option<String> =
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
state = current;
if state.as_deref() == Some("completed") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(state.as_deref(), Some("completed"));
assert_eq!(calls.load(Ordering::SeqCst), 1, "exactly one upstream call");
assert_eq!(
seen_traceparent.lock().unwrap().as_deref(),
Some(correlation.trace_context().traceparent()),
"the durable attempt continues the accepted parent trace"
);
}
/// Story 4.3 end-to-end ATDD: an ambiguous post-checkpoint upstream failure is persisted as
/// `outcome_unknown` (not a retryable failure) and is not redispatched automatically.
#[tokio::test]
async fn task_worker_ambiguous_upstream_error_becomes_outcome_unknown() {
use axum::Router;
use axum::http::StatusCode;
use axum::routing::post;
use crank_community_mcp::TaskWorkerHandler;
use crank_core::{AgentOperationBinding, PlatformApiKeyScope};
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
use std::sync::Arc;
use std::sync::atomic::Ordering;
let registry = common::test_registry().await;
let upstream_app = Router::new().route(
"/crm/leads",
post(|| async { (StatusCode::SERVICE_UNAVAILABLE, "upstream unavailable") }),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(background_policy());
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(&registry, &operation.id, 1).await;
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(
&registry,
"task-worker-uncertain",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let correlation = unsampled_correlation();
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("test master-key identity registered");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let tasks = TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
);
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
"worker-uncertain-key",
&json!({"email": "user@example.com"}),
&correlation,
)
.await
.expect("durable acceptance with executable input");
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build();
let handler = Arc::new(TaskWorkerHandler::from_application(
&tasks,
registry.clone(),
runtime,
crypto,
));
let config = TaskWorkerConfig {
owner: "runtime-worker-uncertain".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let mut state = None;
for _ in 0..200 {
tokio::time::sleep(Duration::from_millis(50)).await;
let current: Option<String> =
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
state = current;
if state.as_deref() == Some("outcome_unknown") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(state.as_deref(), Some("outcome_unknown"));
// No automatic retry: another authoritative scan never redispaches the uncertain attempt.
assert!(
store
.claim_batch("worker_after", 4, 10_000)
.await
.unwrap()
.is_empty()
);
}
/// Story 4.3 ATDD: graceful stop stops new claims but lets an active attempt reach its fenced
/// terminal commit within the 30s budget.
#[tokio::test]
async fn task_worker_graceful_stop_drains_active_attempt() {
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let registry = common::test_registry().await;
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_drain".to_owned(),
platform_api_key_id: None,
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 1,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: "s".repeat(64),
input_fingerprint: "d".repeat(64),
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&crank_core::TraceContext::parse(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
)
.unwrap(),
),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
};
let accepted = store.accept(request).await.expect("PG accepts");
struct SlowHandler {
started: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl TaskHandler for SlowHandler {
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
Ok(())
}
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
self.started.fetch_add(1, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(500)).await;
commit_attempt(&context, completed_outcome(json!({"ok": true}))).await
}
}
let started = Arc::new(AtomicUsize::new(0));
let handler: Arc<dyn TaskHandler> = Arc::new(SlowHandler {
started: started.clone(),
});
let config = TaskWorkerConfig {
owner: "runtime-worker-drain".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(20),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
for _ in 0..100 {
if started.load(Ordering::SeqCst) > 0 {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
assert_eq!(started.load(Ordering::SeqCst), 1, "attempt started");
let stop_requested = std::time::Instant::now();
stop_flag.store(true, Ordering::SeqCst);
tokio::time::timeout(Duration::from_secs(30), worker_task)
.await
.expect("worker stops inside the 30s budget")
.expect("worker exits cleanly")
.expect("worker reports no error");
assert!(stop_requested.elapsed() <= Duration::from_secs(30));
let access = TaskAccess {
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_drain".to_owned(),
};
let record = store.read(&access, accepted.task_id).await.unwrap();
assert_eq!(record.state, TaskState::Completed);
}
+5
View File
@@ -1,5 +1,10 @@
FROM node:22-alpine AS build
# The CI host uplink drops TLS 1.3 ClientHello carrying the hybrid
# X25519MLKEM768 key share; pin the npm stage to TLS 1.2 until that network
# defect is fixed. Certificates, SRI and the lockfile are unaffected.
ENV NODE_OPTIONS=--tls-max-v1.2
WORKDIR /app
COPY apps/ui/package.json apps/ui/package-lock.json ./
+5
View File
@@ -21,6 +21,7 @@ crank-schema = { path = "../crank-schema" }
crank-trace = { path = "../crank-trace" }
futures-util = "0.3"
reqwest.workspace = true
serde.workspace = true
serde_json.workspace = true
sha2.workspace = true
@@ -31,7 +32,11 @@ tokio = { workspace = true, features = ["sync"] }
tracing.workspace = true
uuid.workspace = true
# Typed Tools dispatch uses the SDK; its HTTP transport remains test-only.
rmcp = { version = "=3.9.0" }
[dev-dependencies]
rmcp = { version = "=3.9.0", features = ["transport-streamable-http-server"] }
crank-mapping = { path = "../crank-mapping" }
crank-test-support = { path = "../crank-test-support" }
metrics-util = "0.20.4"
+4 -1
View File
@@ -272,7 +272,10 @@ fn record_unknown_approval_access_denial(reason: &'static str) {
);
}
fn allows_scope(scopes: &[PlatformApiKeyScope], required_scope: PlatformApiKeyScope) -> bool {
pub(super) fn allows_scope(
scopes: &[PlatformApiKeyScope],
required_scope: PlatformApiKeyScope,
) -> bool {
match required_scope {
PlatformApiKeyScope::Read => scopes.iter().any(|scope| {
matches!(
+15 -2
View File
@@ -34,6 +34,19 @@ pub(crate) mod lifecycle;
mod metrics;
mod request;
mod response;
mod rmcp_dispatch;
#[cfg(test)]
mod rmcp_http;
#[cfg(test)]
#[path = "rmcp_http_tests.rs"]
mod rmcp_http_tests;
#[cfg(test)]
mod rmcp_session_tests;
#[cfg(test)]
mod rmcp_tools;
#[cfg(test)]
#[path = "rmcp_tools_tests.rs"]
mod rmcp_tools_tests;
mod stages;
mod tool_resolution;
pub(crate) use self::approval_policy::{
@@ -51,15 +64,15 @@ mod auth_resolution;
mod dispatch;
mod endpoint;
mod legacy_dispatch;
mod task_worker;
mod tasks;
use self::approvals::{
approve_request, deny_request, get_approval_request, list_pending_approvals,
};
pub(super) use self::auth_resolution::resolve_operation_auth;
use self::endpoint::{mcp_delete, mcp_get, mcp_post};
pub use self::task_worker::{TaskWorkerHandler, VerifiedIdempotentWrite};
pub use self::tasks::{TaskApplication, TaskApplicationError, TaskPrincipal};
// Test-only import surface: the BUILD-internal `app/tests.rs` module reaches these two
// through `super`, and the plain lib build does not use them (same pattern as
// `#[cfg(test)] use invocation_history::observe_invocation_history_outcome;` above).
@@ -19,18 +19,31 @@ pub(crate) async fn resolve_operation_auth(
workspace_id: &crank_core::WorkspaceId,
execution_config: &crank_core::ExecutionConfig,
) -> Result<Option<ResolvedAuth>, RuntimeError> {
if execution_config.auth_profile_ref.is_none() {
return Ok(None);
}
let span = Stage::AuthResolve.span();
let result = resolve_runtime_auth_for_task(
resolve_operation_auth_with(
&state.registry,
&state.secret_crypto,
workspace_id,
execution_config,
)
.instrument(span.clone())
.await;
.await
}
/// Same resolution for callers that hold the authoritative registry/secret-crypto pair without the
/// HTTP application state (the durable Task worker).
pub(crate) async fn resolve_operation_auth_with(
registry: &PostgresRegistry,
secret_crypto: &SecretCrypto,
workspace_id: &crank_core::WorkspaceId,
execution_config: &crank_core::ExecutionConfig,
) -> Result<Option<ResolvedAuth>, RuntimeError> {
if execution_config.auth_profile_ref.is_none() {
return Ok(None);
}
let span = Stage::AuthResolve.span();
let result =
resolve_runtime_auth_for_task(registry, secret_crypto, workspace_id, execution_config)
.instrument(span.clone())
.await;
match &result {
Ok(_) => StageOutcome::Success.record(&span),
Err(_) => {
+39 -32
View File
@@ -7,6 +7,9 @@
use super::request::{InitializeParams, ToolCallParams};
use super::response::take_confirmation_token;
use super::rmcp_dispatch::{
handle_resources_command, handle_tools_command, list_resource_templates,
};
use crate::{
auth::VerifiedMachineCredential,
jsonrpc::{
@@ -18,8 +21,7 @@ use crate::{
PromptGetCommand, PromptsCommand, PromptsListCommand, PromptsOutcome, RequestIdentity,
ResourceReadCommand, ResourcesCommand, ResourcesListCommand, ResourcesOutcome,
ServiceContext, ToolCallCommand, ToolsCommand, ToolsListCommand, ToolsOutcome,
declared_mcp_method, discover_result, handle_completion_command, handle_prompts_command,
handle_resources_command, handle_tools_command, validate_modern_request_name,
declared_mcp_method, discover_result, validate_modern_request_name,
},
session::SessionState,
tool_error::execution_locale,
@@ -42,7 +44,7 @@ use super::response::{
resource_templates_list_result, resources_list_result, tools_list_result,
};
use super::{AgentRoutePath, AppState, TRANSPORT_SESSION_TTL_MS};
use crate::protocol::{ResourceTemplatesListError, list_resource_templates};
use crate::protocol::ResourceTemplatesListError;
const SERVER_NAME: &str = "crank-mcp-server";
const SERVER_VERSION: &str = env!("CARGO_PKG_VERSION");
@@ -160,19 +162,18 @@ pub(super) async fn handle_modern_request(
)
.await;
match outcome {
ToolsOutcome::List { tools, next_cursor } => transport_response(
StatusCode::OK,
jsonrpc_result(
request_id(message),
crate::protocol::project_modern_cacheable_result(
version,
tools_list_result(tools, next_cursor),
ToolsOutcome::List { tools, next_cursor } => {
match tools_list_result(tools, next_cursor, version) {
Ok(result) => transport_response(
StatusCode::OK,
jsonrpc_result(request_id(message), result),
response_mode,
None,
Some(version),
),
),
response_mode,
None,
Some(version),
),
Err(error) => internal_jsonrpc_error(message, error),
}
}
ToolsOutcome::InvalidParams(message_text) => transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, message_text),
@@ -286,10 +287,10 @@ pub(super) async fn handle_modern_request(
StatusCode::OK,
jsonrpc_result(
request_id(message),
crate::protocol::project_modern_cacheable_result(
version,
resources_list_result(resources, next_cursor),
),
match resources_list_result(resources, next_cursor, version) {
Ok(result) => result,
Err(error) => return internal_jsonrpc_error(message, error),
},
),
response_mode,
None,
@@ -345,10 +346,10 @@ pub(super) async fn handle_modern_request(
StatusCode::OK,
jsonrpc_result(
request_id(message),
crate::protocol::project_modern_cacheable_result(
version,
resource_read_result(contents),
),
match resource_read_result(contents, version) {
Ok(result) => result,
Err(error) => return internal_jsonrpc_error(message, error),
},
),
response_mode,
None,
@@ -434,10 +435,10 @@ pub(super) async fn handle_modern_request(
StatusCode::OK,
jsonrpc_result(
request_id(message),
crate::protocol::project_modern_cacheable_result(
version,
resource_templates_list_result(templates, next_cursor),
),
match resource_templates_list_result(templates, next_cursor, version) {
Ok(result) => result,
Err(error) => return internal_jsonrpc_error(message, error),
},
),
response_mode,
None,
@@ -466,7 +467,7 @@ pub(super) async fn handle_modern_request(
);
}
};
let outcome = handle_prompts_command(
let outcome = super::rmcp_dispatch::handle_prompts_command(
state,
resource_service_context(path, credential, transport_correlation, message),
version,
@@ -483,7 +484,10 @@ pub(super) async fn handle_modern_request(
request_id(message),
crate::protocol::project_modern_cacheable_result(
version,
prompts_list_result(prompts, next_cursor),
match prompts_list_result(prompts, next_cursor) {
Ok(result) => result,
Err(error) => return internal_jsonrpc_error(message, error),
},
),
),
response_mode,
@@ -525,7 +529,7 @@ pub(super) async fn handle_modern_request(
if let Err(response) = validate_modern_request_name(message, headers, &command.name) {
return *response;
}
let outcome = handle_prompts_command(
let outcome = super::rmcp_dispatch::handle_prompts_command(
state,
resource_service_context(path, credential, transport_correlation, message),
version,
@@ -542,7 +546,10 @@ pub(super) async fn handle_modern_request(
request_id(message),
crate::protocol::project_modern_result(
version,
prompt_get_result(description, messages),
match prompt_get_result(description, messages) {
Ok(result) => result,
Err(error) => return internal_jsonrpc_error(message, error),
},
),
),
response_mode,
@@ -616,7 +623,7 @@ pub(super) async fn handle_modern_request(
);
}
};
let outcome = handle_completion_command(
let outcome = super::rmcp_dispatch::handle_completion_command(
state,
resource_service_context(path, credential, transport_correlation, message),
version,
@@ -51,9 +51,87 @@ pub(crate) struct CounterSet {
era_modern: AtomicUsize,
era_legacy: AtomicUsize,
tools_service: AtomicUsize,
sdk_tools_dispatch: AtomicUsize,
sdk_tools_list_result: AtomicUsize,
sdk_tools_call_result: AtomicUsize,
sdk_resources_dispatch: AtomicUsize,
sdk_resources_list_result: AtomicUsize,
sdk_resources_read_result: AtomicUsize,
sdk_resource_templates_list_result: AtomicUsize,
sdk_prompts_dispatch: AtomicUsize,
sdk_prompts_list_result: AtomicUsize,
sdk_prompts_get_result: AtomicUsize,
sdk_completion_dispatch: AtomicUsize,
sdk_completion_result: AtomicUsize,
sdk_logging_dispatch: AtomicUsize,
sdk_logging_level: AtomicUsize,
sdk_logging_notification: AtomicUsize,
}
impl CounterSet {
pub(crate) fn sdk_logging_dispatch(&self) -> usize {
self.sdk_logging_dispatch.load(Ordering::SeqCst)
}
pub(crate) fn sdk_logging_level(&self) -> usize {
self.sdk_logging_level.load(Ordering::SeqCst)
}
pub(crate) fn sdk_logging_notification(&self) -> usize {
self.sdk_logging_notification.load(Ordering::SeqCst)
}
pub(crate) fn sdk_completion_dispatch(&self) -> usize {
self.sdk_completion_dispatch.load(Ordering::SeqCst)
}
pub(crate) fn sdk_completion_result(&self) -> usize {
self.sdk_completion_result.load(Ordering::SeqCst)
}
pub(crate) fn sdk_prompts_dispatch(&self) -> usize {
self.sdk_prompts_dispatch.load(Ordering::SeqCst)
}
pub(crate) fn sdk_prompts_list_result(&self) -> usize {
self.sdk_prompts_list_result.load(Ordering::SeqCst)
}
pub(crate) fn sdk_prompts_get_result(&self) -> usize {
self.sdk_prompts_get_result.load(Ordering::SeqCst)
}
pub(crate) fn sdk_resources_dispatch(&self) -> usize {
self.sdk_resources_dispatch.load(Ordering::SeqCst)
}
pub(crate) fn sdk_resources_list_result(&self) -> usize {
self.sdk_resources_list_result.load(Ordering::SeqCst)
}
pub(crate) fn sdk_resources_read_result(&self) -> usize {
self.sdk_resources_read_result.load(Ordering::SeqCst)
}
pub(crate) fn sdk_resource_templates_list_result(&self) -> usize {
self.sdk_resource_templates_list_result
.load(Ordering::SeqCst)
}
/// SDK typed Tools dispatch entries, separate from the existing era/service snapshot.
pub(crate) fn sdk_tools_dispatch(&self) -> usize {
self.sdk_tools_dispatch.load(Ordering::SeqCst)
}
/// Successfully serialized SDK ListToolsResult values, not dispatch entries.
pub(crate) fn sdk_tools_list_result(&self) -> usize {
self.sdk_tools_list_result.load(Ordering::SeqCst)
}
/// Successfully serialized SDK CallToolResult values, not dispatch entries.
pub(crate) fn sdk_tools_call_result(&self) -> usize {
self.sdk_tools_call_result.load(Ordering::SeqCst)
}
/// Read the counters of one observed request as one comparable snapshot.
pub(crate) fn snapshot(&self) -> Snapshot {
Snapshot {
@@ -134,6 +212,96 @@ pub(crate) fn tools_service_enter() {
increment(|counters| &counters.tools_service);
}
/// `SDK_TOOLS_DISPATCH`: entered the real SDK typed Tools dispatch, not just admission.
pub(crate) fn sdk_tools_dispatch_enter() {
increment(|counters| &counters.sdk_tools_dispatch);
}
/// Mark only after serializing a real SDK ListToolsResult successfully.
pub(crate) fn sdk_tools_list_result_serialized() {
increment(|counters| &counters.sdk_tools_list_result);
}
/// Mark only after serializing a real SDK CallToolResult successfully.
pub(crate) fn sdk_tools_call_result_serialized() {
increment(|counters| &counters.sdk_tools_call_result);
}
/// Mark only after converting a Resources request through the real SDK typed path.
pub(crate) fn sdk_resources_dispatch_enter() {
increment(|counters| &counters.sdk_resources_dispatch);
}
/// Mark only after successfully serializing a real SDK ListResourcesResult.
pub(crate) fn sdk_resources_list_result_serialized() {
increment(|counters| &counters.sdk_resources_list_result);
}
/// Mark only after successfully serializing a real SDK ReadResourceResult.
pub(crate) fn sdk_resources_read_result_serialized() {
increment(|counters| &counters.sdk_resources_read_result);
}
/// Mark only after successfully serializing a real SDK ListResourceTemplatesResult.
pub(crate) fn sdk_resource_templates_list_result_serialized() {
increment(|counters| &counters.sdk_resource_templates_list_result);
}
#[cfg(test)]
#[path = "../rmcp_resources_runtime_tests.rs"]
mod rmcp_resources_runtime_tests;
/// Mark only after converting a Prompts request through the real SDK typed path.
pub(crate) fn sdk_prompts_dispatch_enter() {
increment(|counters| &counters.sdk_prompts_dispatch);
}
/// Mark only after successfully serializing a real SDK ListPromptsResult.
pub(crate) fn sdk_prompts_list_result_serialized() {
increment(|counters| &counters.sdk_prompts_list_result);
}
/// Mark only after successfully serializing a real SDK GetPromptResult.
pub(crate) fn sdk_prompts_get_result_serialized() {
increment(|counters| &counters.sdk_prompts_get_result);
}
#[cfg(test)]
#[path = "../rmcp_prompts_runtime_tests.rs"]
mod rmcp_prompts_runtime_tests;
/// Mark only after constructing real SDK CompleteRequestParams.
pub(crate) fn sdk_completion_dispatch_enter() {
increment(|counters| &counters.sdk_completion_dispatch);
}
/// Mark only after successfully serializing real SDK CompleteResult/CompletionInfo.
pub(crate) fn sdk_completion_result_serialized() {
increment(|counters| &counters.sdk_completion_result);
}
#[cfg(test)]
#[path = "../rmcp_completion_runtime_tests.rs"]
mod rmcp_completion_runtime_tests;
/// Mark only after constructing the real SDK typed legacy logging request.
pub(crate) fn sdk_logging_dispatch_enter() {
increment(|counters| &counters.sdk_logging_dispatch);
}
/// Mark only after successfully decoding a real SDK LoggingLevel at either input.
pub(crate) fn sdk_logging_level_parsed() {
increment(|counters| &counters.sdk_logging_level);
}
/// Mark only after serializing real SDK LoggingMessageNotificationParam params, not an envelope.
pub(crate) fn sdk_logging_notification_serialized() {
increment(|counters| &counters.sdk_logging_notification);
}
#[path = "../rmcp_logging_runtime_tests.rs"]
mod rmcp_logging_runtime_tests;
fn increment(counter: impl Fn(&CounterSet) -> &AtomicUsize) {
let _ = OBSERVER.try_with(|counters| {
counter(counters).fetch_add(1, Ordering::SeqCst);
@@ -144,7 +312,7 @@ fn increment(counter: impl Fn(&CounterSet) -> &AtomicUsize) {
/// execution-boundary reason as the tests below).
#[cfg(test)]
#[path = "../dispatch_observer_support.rs"]
mod support;
pub(crate) mod support;
/// The behavioural section 7.2 harness: real app, real PostgreSQL, one counted loopback upstream and
/// one request-scoped observer per request. Its file lives outside `src/app/` only because the
@@ -153,3 +321,7 @@ mod support;
#[cfg(test)]
#[path = "../dispatch_observer_harness.rs"]
mod tests;
#[cfg(test)]
#[path = "../rmcp_runtime_tests.rs"]
mod rmcp_runtime_tests;
@@ -24,6 +24,9 @@ use super::response::{
resource_templates_list_result, resources_list_result, take_confirmation_token,
tools_list_result,
};
use super::rmcp_dispatch::{
handle_resources_command, handle_tools_command, list_resource_templates,
};
use super::{AgentRoutePath, AppState};
use crate::{
auth::VerifiedMachineCredential,
@@ -36,9 +39,7 @@ use crate::{
EVENT_LEVEL_APPLIED, INVALID_LEVEL, Kind, LoggingOutcome, MethodHandler, PromptGetCommand,
PromptsCommand, PromptsListCommand, PromptsOutcome, RequestIdentity, ResourceReadCommand,
ResourcesCommand, ResourcesListCommand, ResourcesOutcome, ServiceContext, SetLevelCommand,
ToolCallCommand, ToolsCommand, ToolsListCommand, ToolsOutcome, handle_completion_command,
handle_logging_command, handle_prompts_command, handle_resources_command,
handle_tools_command,
ToolCallCommand, ToolsCommand, ToolsListCommand, ToolsOutcome, handle_logging_command,
},
tool_error::execution_locale,
transport::{ResponseMode, session_id_from_headers, transport_response},
@@ -143,13 +144,18 @@ pub(super) async fn handle_legacy_request(
)
.await;
match outcome {
ToolsOutcome::List { tools, next_cursor } => transport_response(
StatusCode::OK,
jsonrpc_result(request_id(message), tools_list_result(tools, next_cursor)),
response_mode,
None,
Some(&session.protocol_version),
),
ToolsOutcome::List { tools, next_cursor } => {
match tools_list_result(tools, next_cursor, &session.protocol_version) {
Ok(result) => transport_response(
StatusCode::OK,
jsonrpc_result(request_id(message), result),
response_mode,
None,
Some(&session.protocol_version),
),
Err(error) => internal_jsonrpc_error(message, error),
}
}
ToolsOutcome::InvalidParams(message_text) => transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, message_text),
@@ -278,7 +284,14 @@ pub(super) async fn handle_legacy_request(
StatusCode::OK,
jsonrpc_result(
request_id(message),
resources_list_result(resources, next_cursor),
match resources_list_result(
resources,
next_cursor,
&session.protocol_version,
) {
Ok(result) => result,
Err(error) => return internal_jsonrpc_error(message, error),
},
),
response_mode,
None,
@@ -335,7 +348,13 @@ pub(super) async fn handle_legacy_request(
match outcome {
ResourcesOutcome::Read { contents } => transport_response(
StatusCode::OK,
jsonrpc_result(request_id(message), resource_read_result(contents)),
jsonrpc_result(
request_id(message),
match resource_read_result(contents, &session.protocol_version) {
Ok(result) => result,
Err(error) => return internal_jsonrpc_error(message, error),
},
),
response_mode,
None,
Some(&session.protocol_version),
@@ -394,7 +413,7 @@ pub(super) async fn handle_legacy_request(
);
}
};
let (templates, next_cursor) = match crate::protocol::list_resource_templates(
let (templates, next_cursor) = match list_resource_templates(
&state,
&path.workspace_slug,
&path.agent_slug,
@@ -425,7 +444,14 @@ pub(super) async fn handle_legacy_request(
StatusCode::OK,
jsonrpc_result(
request_id(message),
resource_templates_list_result(templates, next_cursor),
match resource_templates_list_result(
templates,
next_cursor,
&session.protocol_version,
) {
Ok(result) => result,
Err(error) => return internal_jsonrpc_error(message, error),
},
),
response_mode,
None,
@@ -458,7 +484,7 @@ pub(super) async fn handle_legacy_request(
);
}
};
let outcome = handle_prompts_command(
let outcome = super::rmcp_dispatch::handle_prompts_command(
&state,
ServiceContext {
workspace_slug: &session.workspace_slug,
@@ -480,7 +506,10 @@ pub(super) async fn handle_legacy_request(
StatusCode::OK,
jsonrpc_result(
request_id(message),
prompts_list_result(prompts, next_cursor),
match prompts_list_result(prompts, next_cursor) {
Ok(result) => result,
Err(error) => return internal_jsonrpc_error(message, error),
},
),
response_mode,
None,
@@ -520,7 +549,7 @@ pub(super) async fn handle_legacy_request(
);
}
};
let outcome = handle_prompts_command(
let outcome = super::rmcp_dispatch::handle_prompts_command(
&state,
ServiceContext {
workspace_slug: &session.workspace_slug,
@@ -542,7 +571,10 @@ pub(super) async fn handle_legacy_request(
StatusCode::OK,
jsonrpc_result(
request_id(message),
prompt_get_result(description, messages),
match prompt_get_result(description, messages) {
Ok(result) => result,
Err(error) => return internal_jsonrpc_error(message, error),
},
),
response_mode,
None,
@@ -615,7 +647,7 @@ pub(super) async fn handle_legacy_request(
);
}
};
let outcome = handle_completion_command(
let outcome = super::rmcp_dispatch::handle_completion_command(
&state,
ServiceContext {
workspace_slug: &session.workspace_slug,
@@ -37,16 +37,14 @@ use futures_util::stream;
use serde_json::{Number, Value, json};
use tokio::sync::{mpsc, watch};
use super::rmcp_dispatch::handle_tools_command;
use super::{
AppState,
response::{ToolCommandPayload, tool_command_payload, tool_command_response},
};
use crate::{
jsonrpc::request_id,
protocol::{
ContinuationSupport, ServiceContext, ToolCallCommand, ToolsCommand, ToolsOutcome,
handle_tools_command,
},
protocol::{ContinuationSupport, ServiceContext, ToolCallCommand, ToolsCommand, ToolsOutcome},
transport::{ResponseMode, sse_response},
};
@@ -133,7 +131,7 @@ pub(crate) struct ProgressEvent {
/// The pinned `notifications/progress` frame. Both supported revisions declare the same parameters,
/// so the projection is shared; the token is echoed verbatim and never validated as a credential.
fn progress_frame(token: &ProgressToken, event: &ProgressEvent) -> Value {
pub(super) fn progress_frame(token: &ProgressToken, event: &ProgressEvent) -> Value {
json!({
"jsonrpc": "2.0",
"method": "notifications/progress",
+454 -85
View File
@@ -1,9 +1,9 @@
use axum::{http::StatusCode, response::Response};
use rmcp::model::{CacheScope, CallToolResult, ContentBlock, ListToolsResult, ResultType, Tool};
use serde_json::{Value, json};
use crate::{
jsonrpc::{jsonrpc_result, request_id},
protocol::project_modern_result,
tool_error::{ToolErrorContract, tool_error_text, tool_error_value},
transport::{ResponseMode, transport_response},
};
@@ -30,22 +30,7 @@ pub(super) fn success_tool_payload(
protocol_version: &str,
output: Value,
) -> Value {
jsonrpc_result(
request_id(message),
project_modern_result(
protocol_version,
json!({
"content": [
{
"type": "text",
"text": serde_json::to_string_pretty(&output).unwrap_or_else(|_| "{}".to_owned())
}
],
"structuredContent": output,
"isError": false
}),
),
)
call_result_payload(message, protocol_version, sdk_success_tool_result(output))
}
#[cfg(test)]
@@ -69,85 +54,273 @@ pub(super) fn tool_error_payload(
protocol_version: &str,
error: ToolErrorContract,
) -> Value {
let error_message = tool_error_text(&error);
let error_value = tool_error_value(&error);
jsonrpc_result(
request_id(message),
project_modern_result(
protocol_version,
json!({
"content": [
{
"type": "text",
"text": error_message
}
],
"structuredContent": {
"error": error_value
},
"isError": true
}),
),
)
call_result_payload(message, protocol_version, sdk_error_tool_result(error))
}
/// The era-neutral `tools/list` result shape built from the shared command outcome. Object keys end
/// up byte-lexicographically ordered, exactly as before the extraction.
pub(super) fn tools_list_result(tools: Vec<Value>, next_cursor: Option<String>) -> Value {
let mut result = json!({ "tools": tools });
if let Some(next_cursor) = next_cursor {
result["nextCursor"] = Value::String(next_cursor);
}
pub(super) fn sdk_success_tool_result(output: Value) -> CallToolResult {
let text = serde_json::to_string_pretty(&output).expect("JSON value serializes");
let mut result = CallToolResult::success(vec![ContentBlock::text(text)]);
result.structured_content = Some(output);
result
}
/// The era-neutral `resources/list` result shape (Story 3.8). Same pagination wire keys as
/// `tools/list`; `nextCursor` is absent on the last page.
pub(super) fn resources_list_result(resources: Vec<Value>, next_cursor: Option<String>) -> Value {
let mut result = json!({ "resources": resources });
if let Some(next_cursor) = next_cursor {
result["nextCursor"] = Value::String(next_cursor);
}
pub(super) fn sdk_error_tool_result(error: ToolErrorContract) -> CallToolResult {
let mut result = CallToolResult::error(vec![ContentBlock::text(tool_error_text(&error))]);
result.structured_content = Some(json!({ "error": tool_error_value(&error) }));
result
}
/// The era-neutral `resources/read` result shape: exactly the typed `contents[]` the revision
/// converter produced.
pub(super) fn resource_read_result(contents: Value) -> Value {
json!({ "contents": contents })
fn call_result_payload(message: &Value, version: &str, mut result: CallToolResult) -> Value {
result.result_type =
(version == crate::jsonrpc::MODERN_PROTOCOL_VERSION).then_some(ResultType::COMPLETE);
let result = serde_json::to_value(result).expect("SDK call result contains JSON values");
#[cfg(test)]
super::dispatch_observer::sdk_tools_call_result_serialized();
jsonrpc_result(request_id(message), result)
}
pub(super) fn sdk_tools_list_result(
tools: &[Value],
next_cursor: Option<String>,
) -> Result<ListToolsResult, serde_json::Error> {
Ok(ListToolsResult {
tools: tools
.iter()
.cloned()
.map(serde_json::from_value::<Tool>)
.collect::<Result<Vec<_>, _>>()?,
next_cursor,
..Default::default()
})
}
/// Serialize the SDK model, preserving catalog extensions that pinned Tool serde cannot represent.
pub(super) fn tools_list_result(
tools: Vec<Value>,
next_cursor: Option<String>,
version: &str,
) -> Result<Value, serde_json::Error> {
let mut result = sdk_tools_list_result(&tools, next_cursor)?;
let modern = version == crate::jsonrpc::MODERN_PROTOCOL_VERSION;
result.result_type = modern.then_some(ResultType::COMPLETE);
result.ttl_ms = modern.then_some(0);
result.cache_scope = modern.then_some(CacheScope::Private);
let mut result = serde_json::to_value(result)?;
#[cfg(test)]
super::dispatch_observer::sdk_tools_list_result_serialized();
for (raw, serialized) in tools
.iter()
.zip(result["tools"].as_array_mut().expect("SDK tools array"))
{
let serialized = serialized.as_object_mut().expect("SDK Tool object");
for (key, value) in raw.as_object().expect("validated Tool object") {
if matches!(key.as_str(), "annotations" | "icons")
|| !matches!(
key.as_str(),
"name" | "title" | "description" | "inputSchema" | "outputSchema" | "_meta"
)
|| value.is_null()
{
serialized.insert(key.clone(), value.clone());
}
}
}
Ok(result)
}
/// SDK results are serialized before the minimal lossless projection of descriptor extensions.
pub(super) fn resources_list_result(
resources: Vec<Value>,
next_cursor: Option<String>,
version: &str,
) -> Result<Value, serde_json::Error> {
let modern = version == crate::jsonrpc::MODERN_PROTOCOL_VERSION;
let result = rmcp::model::ListResourcesResult {
resources: resources
.iter()
.cloned()
.map(serde_json::from_value)
.collect::<Result<_, _>>()?,
next_cursor,
result_type: modern.then_some(ResultType::COMPLETE),
ttl_ms: modern.then_some(0),
cache_scope: modern.then_some(CacheScope::Private),
meta: None,
};
let mut result = serde_json::to_value(result)?;
#[cfg(test)]
super::dispatch_observer::sdk_resources_list_result_serialized();
preserve_resource_fields(
&resources,
&mut result["resources"],
&[
"uri",
"name",
"title",
"description",
"mimeType",
"size",
"_meta",
],
);
Ok(result)
}
pub(super) fn resource_read_result(
contents: Value,
version: &str,
) -> Result<Value, serde_json::Error> {
let mut result =
rmcp::model::ReadResourceResult::new(serde_json::from_value(contents.clone())?);
let modern = version == crate::jsonrpc::MODERN_PROTOCOL_VERSION;
result.result_type = modern.then_some(ResultType::COMPLETE);
result.ttl_ms = modern.then_some(0);
result.cache_scope = modern.then_some(CacheScope::Private);
let mut result = serde_json::to_value(result)?;
#[cfg(test)]
super::dispatch_observer::sdk_resources_read_result_serialized();
preserve_resource_fields(
contents.as_array().expect("validated contents array"),
&mut result["contents"],
&["uri", "mimeType", "text", "blob", "_meta"],
);
Ok(result)
}
/// The era-neutral `resources/templates/list` result shape.
pub(super) fn resource_templates_list_result(
templates: Vec<Value>,
next_cursor: Option<String>,
) -> Value {
let mut result = json!({ "resourceTemplates": templates });
if let Some(cursor) = next_cursor {
result["nextCursor"] = Value::String(cursor);
}
result
version: &str,
) -> Result<Value, serde_json::Error> {
let modern = version == crate::jsonrpc::MODERN_PROTOCOL_VERSION;
let result = rmcp::model::ListResourceTemplatesResult {
resource_templates: templates
.iter()
.cloned()
.map(serde_json::from_value)
.collect::<Result<_, _>>()?,
next_cursor,
result_type: modern.then_some(ResultType::COMPLETE),
ttl_ms: modern.then_some(0),
cache_scope: modern.then_some(CacheScope::Private),
meta: None,
};
let mut result = serde_json::to_value(result)?;
#[cfg(test)]
super::dispatch_observer::sdk_resource_templates_list_result_serialized();
preserve_resource_fields(
&templates,
&mut result["resourceTemplates"],
&[
"uriTemplate",
"name",
"title",
"description",
"mimeType",
"_meta",
],
);
Ok(result)
}
/// The era-neutral `prompts/list` result shape (Story 3.14). Same pagination wire keys as
/// `resources/list`; `nextCursor` is absent on the last page.
pub(super) fn prompts_list_result(prompts: Vec<Value>, next_cursor: Option<String>) -> Value {
let mut result = json!({ "prompts": prompts });
if let Some(cursor) = next_cursor {
result["nextCursor"] = Value::String(cursor);
// Pinned SDK serde drops unknown fields and optional nulls, including Crank template parameters.
// Treat annotations/icons as opaque too: their nested extensions are not represented by the SDK.
fn preserve_resource_fields(raw: &[Value], serialized: &mut Value, sdk_fields: &[&str]) {
for (raw, serialized) in raw
.iter()
.zip(serialized.as_array_mut().expect("SDK array"))
{
let serialized = serialized.as_object_mut().expect("SDK object");
for (key, value) in raw.as_object().expect("validated resource object") {
if !sdk_fields.contains(&key.as_str()) || value.is_null() {
serialized.insert(key.clone(), value.clone());
}
}
}
result
}
/// The era-neutral `prompts/get` result shape: the rendered `messages[]` and the description of the
/// exact bound Published Prompt Version it came from.
pub(super) fn prompt_get_result(description: Option<String>, messages: Vec<Value>) -> Value {
let mut result = json!({ "messages": messages });
if let Some(description) = description {
result["description"] = Value::String(description);
/// SDK pagination with lossless Crank argument schemas/defaults and optional nulls.
pub(super) fn prompts_list_result(
prompts: Vec<Value>,
next_cursor: Option<String>,
) -> Result<Value, serde_json::Error> {
let result = rmcp::model::ListPromptsResult {
prompts: prompts
.iter()
.cloned()
.map(serde_json::from_value)
.collect::<Result<Vec<rmcp::model::Prompt>, _>>()?,
next_cursor,
result_type: None,
ttl_ms: None,
cache_scope: None,
meta: None,
};
let mut result = serde_json::to_value(result)?;
preserve_resource_fields(
&prompts,
&mut result["prompts"],
&["name", "title", "description", "arguments", "_meta"],
);
for (raw, serialized) in prompts
.iter()
.zip(result["prompts"].as_array_mut().expect("SDK prompts"))
{
if let Some(arguments) = raw["arguments"].as_array() {
preserve_resource_fields(
arguments,
&mut serialized["arguments"],
&["name", "title", "description", "required"],
);
}
}
result
#[cfg(test)]
super::dispatch_observer::sdk_prompts_list_result_serialized();
Ok(result)
}
/// Preserve Crank's system role, which the pinned SDK's user/assistant Role cannot express.
pub(super) fn prompt_get_result(
description: Option<String>,
messages: Vec<Value>,
) -> Result<Value, serde_json::Error> {
let sdk_messages = messages
.iter()
.cloned()
.map(|mut message| {
if message["role"] == "system" {
message["role"] = json!("user");
}
serde_json::from_value::<rmcp::model::PromptMessage>(message)
})
.collect::<Result<Vec<_>, _>>()?;
let mut result = rmcp::model::GetPromptResult::new(sdk_messages);
result.description = description;
// Era-specific resultType/cache projection remains at the existing dispatch boundary.
result.result_type = None;
let mut result = serde_json::to_value(result)?;
preserve_resource_fields(&messages, &mut result["messages"], &["role", "content"]);
for (raw, serialized) in messages
.iter()
.zip(result["messages"].as_array_mut().expect("SDK messages"))
{
if raw["role"] == "system" {
serialized["role"] = raw["role"].clone();
}
// Content extensions/optional nulls are opaque to the pinned SDK.
let content = serialized["content"].as_object_mut().expect("SDK content");
for (key, value) in raw["content"].as_object().expect("validated content") {
if !content.contains_key(key)
|| value.is_null()
|| matches!(key.as_str(), "annotations" | "resource")
{
content.insert(key.clone(), value.clone());
}
}
}
#[cfg(test)]
super::dispatch_observer::sdk_prompts_get_result_serialized();
Ok(result)
}
/// The era-neutral `completion/complete` result shape (Story 3.17): the bounded `values[]`, the
@@ -158,14 +331,18 @@ pub(super) fn completion_result(
total: Option<usize>,
has_more: bool,
) -> Value {
let mut completion = json!({ "values": values });
if let Some(total) = total {
completion["total"] = json!(total);
}
if has_more {
completion["hasMore"] = json!(true);
}
json!({ "completion": completion })
let mut completion = rmcp::model::CompletionInfo::default();
completion.values = values;
// The domain scans at most 1024 candidates, so its exact count fits the SDK's u32.
completion.total = total.map(|total| u32::try_from(total).expect("bounded completion count"));
completion.has_more = has_more.then_some(true);
let mut result = rmcp::model::CompleteResult::new(completion);
// Keep the shared result era-neutral; the existing caller projects modern resultType.
result.result_type = None;
let result = serde_json::to_value(result).expect("SDK completion contains strings and counts");
#[cfg(test)]
super::dispatch_observer::sdk_completion_result_serialized();
result
}
/// The structured `prompts/get` argument rejection (AT-3.14-02): a `-32602` error whose `data`
@@ -310,3 +487,195 @@ pub(super) fn internal_jsonrpc_error(message: &Value, _error: impl std::fmt::Dis
None,
)
}
#[cfg(test)]
mod sdk_result_tests {
use super::*;
use crate::jsonrpc::{LEGACY_PROTOCOL_VERSION, MODERN_PROTOCOL_VERSION};
#[test]
fn prompts_sdk_projection_preserves_typed_extensions_and_system_content() {
let prompt = json!({"name":"p", "description":null, "icons":null,
"x-prompt":{}, "arguments":[{"name":"count", "description":null,
"required":false, "schema":{"type":"integer"}, "default":7}]});
for cursor in [None, Some(String::new()), Some("opaque".to_owned())] {
let result = prompts_list_result(vec![prompt.clone()], cursor.clone()).unwrap();
assert_eq!(result["prompts"], json!([prompt]));
assert_eq!(result.get("nextCursor"), cursor.map(Value::String).as_ref());
assert!(result.get("resultType").is_none());
}
let messages = vec![
json!({"role":"system", "x-message":null,
"content":{"type":"text", "text":"\0é\n", "annotations":null, "x-content":[]}}),
json!({"role":"assistant", "content":{"type":"text", "text":""}}),
];
assert_eq!(
prompt_get_result(None, messages.clone()).unwrap(),
json!({"messages":messages})
);
assert_eq!(
prompts_list_result(vec![], None).unwrap(),
json!({"prompts":[]})
);
assert_eq!(
prompt_get_result(Some(String::new()), vec![]).unwrap(),
json!({"description":"", "messages":[]})
);
assert!(prompts_list_result(vec![json!({"name":42})], None).is_err());
assert!(
prompt_get_result(
None,
vec![json!({"role":"invalid", "content":{"type":"text", "text":"secret"}})]
)
.is_err()
);
}
#[test]
fn resources_sdk_projection_is_lossless_for_templates_contents_and_catalog() {
for version in [LEGACY_PROTOCOL_VERSION, MODERN_PROTOCOL_VERSION] {
for optional in [Value::Null, json!({}), json!({"x-extension":[null,{}]})] {
let raw = json!({"uri":"crank://r","name":"r","description":null,
"mimeType":"text/plain","_meta":optional,"x-resource":[],
"annotations":{"x-nested":null},"icons":[{"src":"icon","x-icon":{}}]});
for cursor in [None, Some(String::new()), Some("opaque".to_owned())] {
let list =
resources_list_result(vec![raw.clone()], cursor.clone(), version).unwrap();
assert_eq!(list["resources"], json!([raw]));
assert_eq!(
list.get("nextCursor"),
cursor.as_ref().map(|s| json!(s)).as_ref()
);
assert_eq!(
list.get("resultType").is_some(),
version == MODERN_PROTOCOL_VERSION
);
assert_eq!(
list.get("ttlMs").is_some(),
version == MODERN_PROTOCOL_VERSION
);
assert_eq!(
list.get("cacheScope").is_some(),
version == MODERN_PROTOCOL_VERSION
);
assert!(list.get("_meta").is_none());
let template = json!({"uriTemplate":"crank://r/{id}","name":"r",
"description":null,"mimeType":null,"_meta":optional,
"parameters":[{"name":"id","required":true,"opaque":null}],
"annotations":{"x-nested":[]},"icons":null});
let list = resource_templates_list_result(
vec![template.clone()],
cursor.clone(),
version,
)
.unwrap();
assert_eq!(list["resourceTemplates"], json!([template]));
assert_eq!(
list.get("nextCursor"),
cursor.as_ref().map(|s| json!(s)).as_ref()
);
}
let contents = json!([
{"uri":"crank://r","mimeType":null,"text":"\0é\n","_meta":optional,"x-content":[]},
{"uri":"crank://b","mimeType":"application/octet-stream","blob":"AP8=","_meta":null}
]);
assert_eq!(
resource_read_result(contents.clone(), version).unwrap()["contents"],
contents
);
}
assert_eq!(
resources_list_result(vec![], None, version).unwrap()["resources"],
json!([])
);
assert_eq!(
resource_templates_list_result(vec![], None, version).unwrap()["resourceTemplates"],
json!([])
);
assert_eq!(
resource_read_result(json!([]), version).unwrap()["contents"],
json!([])
);
assert!(resources_list_result(vec![json!({"uri":42})], None, version).is_err());
assert!(
resource_templates_list_result(vec![json!({"name":"r"})], None, version).is_err()
);
assert!(resource_read_result(json!([{"uri":"r"}]), version).is_err());
}
}
#[test]
fn catalog_projection_preserves_extensions_null_and_empty() {
let descriptor = json!({
"name":"tool", "title":"", "description":null,
"inputSchema":{"type":"object", "x-schema":{"nested":null}},
"outputSchema":{"type":"object", "x-output":[]},
"_meta":{"custom":{"empty":{}}}, "x-tool":{"nested":[]},
"annotations":{"readOnlyHint":true,"x-annotation":{"nested":null}},
"icons":[{"src":"https://example.test/icon", "x-icon":{"nested":[]}}]
});
for version in [LEGACY_PROTOCOL_VERSION, MODERN_PROTOCOL_VERSION] {
for raw in [
descriptor.clone(),
json!({"name":"tool","inputSchema":{},
"title":null,"description":"","outputSchema":null,"_meta":null,
"annotations":null,"icons":null}),
json!({"name":"tool","inputSchema":{},
"annotations":{},"icons":[]}),
] {
for cursor in [None, Some(String::new()), Some("next".to_owned())] {
let value =
tools_list_result(vec![raw.clone()], cursor.clone(), version).unwrap();
assert_eq!(value["tools"], json!([raw]));
assert_eq!(
value.get("nextCursor"),
cursor.as_ref().map(|s| Value::String(s.clone())).as_ref()
);
assert_eq!(
value.get("resultType").is_some(),
version == MODERN_PROTOCOL_VERSION
);
assert_eq!(
value.get("ttlMs").is_some(),
version == MODERN_PROTOCOL_VERSION
);
assert_eq!(
value.get("cacheScope").is_some(),
version == MODERN_PROTOCOL_VERSION
);
}
}
assert_eq!(
tools_list_result(vec![], None, version).unwrap()["tools"],
json!([])
);
assert!(tools_list_result(vec![json!({"name":42})], None, version).is_err());
}
}
#[test]
fn null_success_and_rpc_failures_keep_their_distinct_shapes() {
for version in [LEGACY_PROTOCOL_VERSION, MODERN_PROTOCOL_VERSION] {
let message = json!({"id":9007199254740993_u64});
let payload = success_tool_payload(&message, version, Value::Null);
assert_eq!(payload["id"], message["id"]);
assert_eq!(
payload["result"]["content"],
json!([{"type":"text","text":"null"}])
);
assert_eq!(
payload["result"].get("structuredContent"),
Some(&Value::Null)
);
assert_eq!(payload["result"]["isError"], false);
let correlation = CorrelationContext::generate();
let internal = tool_command_payload(
&message,
version,
&correlation,
ToolsOutcome::Call(ToolCallOutcome::InternalFailure),
);
assert!(matches!(internal, ToolCommandPayload::Internal(_)));
}
}
}
@@ -0,0 +1,181 @@
//! Runtime SDK dispatch; admission, lifecycle and response projection stay with the caller.
use std::sync::Arc;
use rmcp::model::{CallToolRequestParams, PaginatedRequestParams, ReadResourceRequestParams};
use serde_json::Value;
use super::{AppState, lifecycle::ToolCallLifecycle};
use crate::protocol::{
ServiceContext, ToolCallCommand, ToolCallOutcome, ToolsCommand, ToolsListCommand, ToolsOutcome,
};
// Crank extensions are not SDK fields and must survive the typed request boundary unchanged.
enum SdkToolsCommand {
List {
params: PaginatedRequestParams,
limit: Option<usize>,
},
Call {
params: CallToolRequestParams,
confirmation_token: Option<String>,
},
}
pub(super) async fn handle_tools_command(
state: &Arc<AppState>,
context: ServiceContext<'_>,
command: ToolsCommand,
lifecycle: Option<ToolCallLifecycle>,
) -> ToolsOutcome {
let request = match command {
ToolsCommand::List(list) => SdkToolsCommand::List {
params: PaginatedRequestParams::default().with_cursor(list.cursor),
limit: list.limit,
},
ToolsCommand::Call(call) => {
let Value::Object(arguments) = call.arguments else {
return ToolsOutcome::Call(ToolCallOutcome::InvalidParams(
"tool arguments must be an object".to_owned(),
));
};
SdkToolsCommand::Call {
params: CallToolRequestParams::new(call.tool_name).with_arguments(arguments),
confirmation_token: call.confirmation_token,
}
}
};
#[cfg(test)]
super::dispatch_observer::sdk_tools_dispatch_enter();
let command = match request {
SdkToolsCommand::List { params, limit } => ToolsCommand::List(ToolsListCommand {
cursor: params.cursor,
limit,
}),
SdkToolsCommand::Call {
params,
confirmation_token,
} => ToolsCommand::Call(ToolCallCommand {
tool_name: params.name.into_owned(),
arguments: Value::Object(params.arguments.unwrap_or_default()),
confirmation_token,
}),
};
crate::protocol::handle_tools_command(state, context, command, lifecycle).await
}
pub(super) async fn handle_prompts_command(
state: &Arc<AppState>,
context: ServiceContext<'_>,
revision: &str,
command: crate::protocol::PromptsCommand,
) -> crate::protocol::PromptsOutcome {
use crate::protocol::{PromptGetCommand, PromptsCommand, PromptsListCommand};
let command = match command {
PromptsCommand::List(list) => {
let params = PaginatedRequestParams::default().with_cursor(list.cursor);
PromptsCommand::List(PromptsListCommand {
cursor: params.cursor,
limit: list.limit,
})
}
PromptsCommand::Get(get) => {
let mut params = rmcp::model::GetPromptRequestParams::new(get.name);
params.arguments = get.arguments;
PromptsCommand::Get(PromptGetCommand {
name: params.name,
arguments: params.arguments,
})
}
};
#[cfg(test)]
super::dispatch_observer::sdk_prompts_dispatch_enter();
crate::protocol::handle_prompts_command(state, context, revision, command).await
}
pub(super) async fn handle_resources_command(
state: &Arc<AppState>,
context: ServiceContext<'_>,
revision: &str,
command: crate::protocol::ResourcesCommand,
) -> crate::protocol::ResourcesOutcome {
use crate::protocol::{ResourceReadCommand, ResourcesCommand};
let command = match command {
ResourcesCommand::List(list) => ResourcesCommand::List(sdk_resource_page(list)),
ResourcesCommand::Read(read) => {
let params = ReadResourceRequestParams::new(read.uri);
ResourcesCommand::Read(ResourceReadCommand { uri: params.uri })
}
};
#[cfg(test)]
super::dispatch_observer::sdk_resources_dispatch_enter();
crate::protocol::handle_resources_command(state, context, revision, command).await
}
pub(super) async fn list_resource_templates(
state: &Arc<AppState>,
workspace_slug: &str,
agent_slug: &str,
revision: &str,
command: crate::protocol::ResourcesListCommand,
) -> Result<(Vec<Value>, Option<String>), crate::protocol::ResourceTemplatesListError> {
let command = sdk_resource_page(command);
#[cfg(test)]
super::dispatch_observer::sdk_resources_dispatch_enter();
crate::protocol::list_resource_templates(state, workspace_slug, agent_slug, revision, command)
.await
}
pub(super) async fn handle_completion_command(
state: &Arc<AppState>,
context: ServiceContext<'_>,
revision: &str,
command: crate::protocol::CompletionCommand,
) -> crate::protocol::CompletionOutcome {
use crate::protocol::{
CompletionArgument, CompletionCommand, CompletionOutcome, CompletionReference,
};
use rmcp::model::{ArgumentInfo, CompleteRequestParams, Reference};
let reference = match command.reference {
CompletionReference::Prompt { name } => Reference::for_prompt(name),
CompletionReference::Resource { uri } => Reference::for_resource(uri),
};
let params = CompleteRequestParams::new(
reference,
ArgumentInfo::new(command.argument.name, command.argument.value),
);
// SDK context only supports strings; retain arbitrary JSON unchanged for domain validation.
let context_arguments = command.context;
#[cfg(test)]
super::dispatch_observer::sdk_completion_dispatch_enter();
let reference = match params.r#ref {
Reference::Prompt(reference) => CompletionReference::Prompt {
name: reference.name,
},
Reference::Resource(reference) => CompletionReference::Resource { uri: reference.uri },
_ => return CompletionOutcome::InvalidParams("invalid completion/complete parameters"),
};
let command = CompletionCommand {
reference,
argument: CompletionArgument {
name: params.argument.name,
value: params.argument.value,
},
context: context_arguments,
};
crate::protocol::handle_completion_command(state, context, revision, command).await
}
fn sdk_resource_page(
list: crate::protocol::ResourcesListCommand,
) -> crate::protocol::ResourcesListCommand {
let params = PaginatedRequestParams::default().with_cursor(list.cursor);
// Crank's page limit is not an SDK field; retain its existing validation and clamping.
crate::protocol::ResourcesListCommand {
cursor: params.cursor,
limit: list.limit,
}
}
@@ -0,0 +1,197 @@
//! Isolated single-Agent loopback experiment. Not installed in the production router.
//! Each instance owns one route and one SDK manager; no durable store is replaced.
use super::{
AgentRoutePath, AppState,
lifecycle::InFlightScope,
metrics::ActiveStreamGuard,
rmcp_tools::{RmcpRequestIdentity, RmcpTools},
stages::{enforce_traced_rate_limit, require_traced_machine_access},
};
use crate::{
protocol::{self, ContinuationSupport, RequestEra},
transport,
};
use axum::{
Router,
body::{Body, to_bytes},
extract::Request,
http::{Method, StatusCode},
response::{IntoResponse, Response},
};
use crank_core::PlatformApiKeyScope;
use futures_util::StreamExt;
use rmcp::transport::streamable_http_server::{
StreamableHttpServerConfig, StreamableHttpService, session::local::LocalSessionManager,
};
use serde_json::Value;
use std::sync::Arc;
const BODY_LIMIT: usize = 2 * 1024 * 1024;
pub(super) fn router(
state: Arc<AppState>,
path: AgentRoutePath,
active: Arc<std::sync::atomic::AtomicUsize>,
) -> Router {
let handler_state = state.clone();
let mut config = StreamableHttpServerConfig::default().with_json_response(true);
config.max_request_body_bytes = BODY_LIMIT;
let shutdown = Arc::new(config.cancellation_token.clone().drop_guard());
let mut manager = LocalSessionManager::default();
manager.session_config.channel_capacity = 16;
let service = StreamableHttpService::new(
move || {
Ok(RmcpTools {
state: handler_state.clone(),
active: active.clone(),
})
},
Arc::new(manager),
config,
);
let route = format!("/v1/{}/{}", path.workspace_slug, path.agent_slug);
Router::new()
.route(
&route,
axum::routing::any(move |request: Request| {
let state = state.clone();
let path = path.clone();
let service = service.clone();
let shutdown = shutdown.clone();
async move {
let _shutdown = shutdown;
let (mut parts, body) = request.into_parts();
if let Err(status) =
transport::validate_origin(&state.allowed_origins, &parts.headers)
{
return status.into_response();
}
let bytes = match to_bytes(body, BODY_LIMIT).await {
Ok(bytes) => bytes,
Err(_) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(),
};
let message: Value = if parts.method == Method::POST {
match serde_json::from_slice(&bytes) {
Ok(message) => message,
Err(_) => return StatusCode::BAD_REQUEST.into_response(),
}
} else {
Value::Null
};
let method = message["method"].as_str().unwrap_or("");
let required = if method == "tools/call" {
PlatformApiKeyScope::Write
} else {
PlatformApiKeyScope::Read
};
let credential = match require_traced_machine_access(
&state,
&path,
&parts.headers,
required,
)
.await
{
Ok(credential) => credential,
Err(error) => return error.into_response(),
};
let session = match transport::session_id_from_headers(&parts.headers) {
Ok(session) => session,
Err(_) => {
return protocol::ambiguous_session_carrier_response(
&parts.headers,
&message,
);
}
};
let era = if parts.method == Method::POST {
if let Err(response) = protocol::validate_jsonrpc_envelope(&message) {
return *response;
}
if let Err(status) = transport::negotiate_post_response_mode(&parts.headers)
{
return status.into_response();
}
match protocol::resolve_request_identity(&parts.headers, &message) {
Ok(identity) => identity.era,
Err(response) => return *response,
}
} else {
if let Err(response) = protocol::session_lifecycle_admission(&parts.headers)
{
return *response;
}
RequestEra::Legacy
};
if matches!(era, RequestEra::Modern) && session.is_some() {
return protocol::modern_carrier_conflict_response(&message);
}
if let Err(error) =
enforce_traced_rate_limit(&state, &path, &parts.headers).await
{
return crate::rate_limit::rate_limited_status_response(error);
}
// SDK cancellation is keyed only by request id. Never forward an unchecked cancel.
if method == "notifications/cancelled" {
if let Some(session_id) = session {
state.in_flight.cancel(
&InFlightScope {
session_id,
owner: credential
.platform_api_key_id
.as_ref()
.map(|id| id.as_str())
.unwrap_or("")
.to_owned(),
},
&message["params"]["requestId"],
);
}
return StatusCode::ACCEPTED.into_response();
}
let Some(context) = parts
.extensions
.get::<crate::request_context::RequestContext>()
else {
return StatusCode::INTERNAL_SERVER_ERROR.into_response();
};
let correlation = context.correlation.clone();
parts.extensions.insert(RmcpRequestIdentity {
workspace_slug: path.workspace_slug,
agent_slug: path.agent_slug,
credential,
correlation,
continuation: ContinuationSupport::from_supported(false),
locale: crate::tool_error::execution_locale(&message),
session_id: session,
progress_meta: message.pointer("/params/_meta").cloned(),
sse_acceptable: transport::accepts_event_stream(&parts.headers),
});
// Legacy SDK calls outlive disconnected bodies: both lifetimes share one slot.
let permit = match ActiveStreamGuard::try_acquire(&state.session_slots) {
Ok(permit) => Arc::new(permit),
Err(_) => return StatusCode::TOO_MANY_REQUESTS.into_response(),
};
if method == "tools/call" {
parts.extensions.insert(permit.clone());
}
let response = service
.handle(Request::from_parts(parts, Body::from(bytes)))
.await;
let (parts, body) = response.into_parts();
let stream = futures_util::stream::unfold(
(Box::pin(Body::new(body).into_data_stream()), permit),
|(mut stream, permit)| async move {
stream.next().await.map(|chunk| (chunk, (stream, permit)))
},
);
Response::from_parts(parts, Body::from_stream(stream))
}
}),
)
.layer(axum::middleware::from_fn(
crate::request_context::apply_request_context,
))
}
@@ -0,0 +1,43 @@
//! Behavioural ATDD for rmcp 3.7.0 LocalSessionManager reconnect/restart semantics.
use rmcp::transport::streamable_http_server::session::{
SessionManager, local::LocalSessionManager,
};
#[tokio::test]
async fn rmcp_local_session_manager_preserves_access_to_created_session() {
let manager = LocalSessionManager::default();
let (session_id, _transport) = manager
.create_session()
.await
.expect("local manager must create a session");
assert!(
manager
.has_session(&session_id)
.await
.expect("live manager must answer has_session"),
"a live manager must keep access to a session it created"
);
}
#[tokio::test]
async fn rmcp_fresh_session_manager_without_external_store_rejects_known_session_id() {
let first_manager = LocalSessionManager::default();
let (session_id, _transport) = first_manager
.create_session()
.await
.expect("first local manager must create a session");
let second_manager = LocalSessionManager::default();
assert!(
!second_manager
.has_session(&session_id)
.await
.expect("fresh manager must answer has_session"),
"a fresh in-memory manager must not claim ownership of another manager's session"
);
}
@@ -0,0 +1,242 @@
//! Isolated rmcp 3.7.0 Tools adapter over Crank's existing typed Tools service.
//!
//! This module deliberately does not provide a second command engine. It owns only
//! the verified request identity (`ServiceContext` data) and delegates both
//! `tools/list` and `tools/call` to `handle_tools_command` via rmcp's
//! `ServerHandler` trait. HTTP admission must supply identity for each request.
use std::sync::Arc;
use crank_core::{CorrelationContext, ExecutionLocale};
use crate::app::{AppState, lifecycle::ToolCallLifecycle};
use crate::auth::VerifiedMachineCredential;
use crate::protocol::{
ContinuationSupport, ServiceContext, ToolCallCommand, ToolsCommand, ToolsListCommand,
ToolsOutcome, handle_tools_command,
};
/// Long-lived handler: never stores credentials obtained on initialization.
#[derive(Clone)]
pub(super) struct RmcpTools {
pub(super) state: Arc<AppState>,
pub(super) active: Arc<std::sync::atomic::AtomicUsize>,
}
struct ActiveCall(Arc<std::sync::atomic::AtomicUsize>);
impl Drop for ActiveCall {
fn drop(&mut self) {
self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
}
}
/// Server-created identity attached to the individual admitted HTTP request.
#[derive(Clone)]
pub(super) struct RmcpRequestIdentity {
pub(super) workspace_slug: String,
pub(super) agent_slug: String,
pub(super) credential: VerifiedMachineCredential,
pub(super) correlation: CorrelationContext,
pub(super) continuation: ContinuationSupport,
pub(super) locale: ExecutionLocale,
pub(super) session_id: Option<String>,
pub(super) progress_meta: Option<serde_json::Value>,
pub(super) sse_acceptable: bool,
}
impl RmcpRequestIdentity {
/// Build the neutral `ServiceContext` borrowed from this adapter's owned data.
pub(super) fn service_context(&self) -> ServiceContext<'_> {
ServiceContext {
workspace_slug: &self.workspace_slug,
agent_slug: &self.agent_slug,
credential: &self.credential,
correlation: &self.correlation,
continuation: self.continuation,
locale: self.locale,
}
}
}
impl RmcpTools {
/// Delegate one neutral Tools command to the existing shared service.
pub(super) async fn handle(
&self,
identity: &RmcpRequestIdentity,
command: ToolsCommand,
lifecycle: Option<ToolCallLifecycle>,
) -> ToolsOutcome {
handle_tools_command(&self.state, identity.service_context(), command, lifecycle).await
}
pub(super) async fn list_tools(
&self,
identity: &RmcpRequestIdentity,
command: ToolsListCommand,
lifecycle: Option<ToolCallLifecycle>,
) -> ToolsOutcome {
self.handle(identity, ToolsCommand::List(command), lifecycle)
.await
}
pub(super) async fn call_tool(
&self,
identity: &RmcpRequestIdentity,
command: ToolCallCommand,
lifecycle: Option<ToolCallLifecycle>,
) -> ToolsOutcome {
self.handle(identity, ToolsCommand::Call(command), lifecycle)
.await
}
}
use rmcp::ErrorData as McpError;
use rmcp::model::{CallToolRequestParams, ListToolsResult, PaginatedRequestParams};
use rmcp::service::RequestContext;
use rmcp::{RoleServer, ServerHandler};
fn request_identity(context: &RequestContext<RoleServer>) -> Result<RmcpRequestIdentity, McpError> {
context
.extensions
.get::<axum::http::request::Parts>()
.and_then(|parts| parts.extensions.get::<RmcpRequestIdentity>())
.cloned()
.ok_or_else(|| McpError::internal_error("missing authenticated request context", None))
}
impl ServerHandler for RmcpTools {
fn supported_protocol_versions(
&self,
) -> std::borrow::Cow<'static, [rmcp::model::ProtocolVersion]> {
std::borrow::Cow::Borrowed(&[
rmcp::model::ProtocolVersion::V_2025_11_25,
rmcp::model::ProtocolVersion::V_2026_07_28,
])
}
fn get_info(&self) -> rmcp::model::ServerConfig {
let mut info = rmcp::model::ServerConfig::default();
info.capabilities = rmcp::model::ServerCapabilities::builder()
.enable_tools()
.build();
info
}
fn list_tools(
&self,
request: Option<PaginatedRequestParams>,
context: RequestContext<RoleServer>,
) -> impl std::future::Future<Output = Result<ListToolsResult, McpError>> + Send + '_ {
let request = request.unwrap_or_default();
async move {
let identity = request_identity(&context)?;
let cursor = request.cursor;
let command = crate::protocol::ToolsListCommand {
cursor,
limit: None,
};
match self.list_tools(&identity, command, None).await {
crate::protocol::ToolsOutcome::List { tools, next_cursor } => {
super::response::sdk_tools_list_result(&tools, next_cursor)
.map_err(|_| McpError::internal_error("invalid tool catalog entry", None))
}
crate::protocol::ToolsOutcome::InvalidParams(message) => {
Err(McpError::invalid_params(message, None))
}
_ => Err(McpError::internal_error("crate catalog unavailable", None)),
}
}
}
async fn call_tool(
&self,
request: CallToolRequestParams,
context: RequestContext<RoleServer>,
) -> Result<rmcp::model::CallToolResponse, McpError> {
let identity = request_identity(&context)?;
let arguments = request
.arguments
.map(serde_json::Value::Object)
.unwrap_or_else(|| serde_json::json!({}));
let command = crate::protocol::ToolCallCommand {
tool_name: request.name.to_string(),
arguments,
confirmation_token: None,
};
let token = identity
.sse_acceptable
.then(|| super::lifecycle::token_from_meta(identity.progress_meta.as_ref()))
.flatten();
// Observe future cleanup directly in the isolated loopback oracle.
self.active
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let _active = ActiveCall(self.active.clone());
let (mut lifecycle, mut events) = ToolCallLifecycle::new(token.clone());
let entry = identity.session_id.as_ref().and_then(|session_id| {
self.state.in_flight.register(
&super::lifecycle::InFlightScope {
session_id: session_id.clone(),
owner: identity
.credential
.platform_api_key_id
.as_ref()
.map(|id| id.as_str())
.unwrap_or("")
.to_owned(),
},
&serde_json::to_value(&context.id).expect("request id serializes"),
)
});
if let Some(entry) = &entry {
lifecycle = lifecycle.with_cancel(entry.cancel_signal());
}
let call = self.call_tool(&identity, command, Some(lifecycle.clone()));
tokio::pin!(call);
let outcome = loop {
tokio::select! {
biased;
event = async { match events.as_mut() { Some(events) => events.recv().await, None => std::future::pending().await } } => {
if let Some(event) = event {
if let Some(token) = &token {
let params = serde_json::from_value(super::lifecycle::progress_frame(token, &event)["params"].clone()).map_err(|_| McpError::internal_error("invalid progress event", None))?;
let _ = context.peer.notify_progress(params).await;
}
} else { events = None; }
}
outcome = &mut call => break outcome,
_ = lifecycle.cancelled() => { return Err(McpError::internal_error("request cancelled", None)); },
_ = context.ct.cancelled() => { return Err(McpError::internal_error("request cancelled", None)); },
}
};
match outcome {
crate::protocol::ToolsOutcome::Call(outcome) => match outcome {
crate::protocol::ToolCallOutcome::Completed(value)
| crate::protocol::ToolCallOutcome::ApprovalPending(value) => {
Ok(super::response::sdk_success_tool_result(value).into())
}
crate::protocol::ToolCallOutcome::ApprovalUnavailable(error) => {
Ok(super::response::sdk_error_tool_result(
crate::tool_error::control_tool_error_contract(
error,
identity.correlation.request_id().as_str(),
identity.correlation.trace_id().as_str(),
identity.locale,
),
)
.into())
}
crate::protocol::ToolCallOutcome::InvalidParams(message) => {
Err(McpError::invalid_params(message, None))
}
crate::protocol::ToolCallOutcome::Error(contract) => {
Ok(super::response::sdk_error_tool_result(contract).into())
}
crate::protocol::ToolCallOutcome::InternalFailure => {
Err(McpError::internal_error("tool execution failed", None))
}
},
crate::protocol::ToolsOutcome::InvalidParams(message) => {
Err(McpError::invalid_params(message, None))
}
_ => Err(McpError::internal_error("crate catalog unavailable", None)),
}
}
}
@@ -0,0 +1,381 @@
//! Application-level durable Task handler and attempt authority (Story 4.3/4.4).
//!
//! The handler owns everything that touches product authority: the exact published pin, the typed
//! Platform API key, the approval policy, the encrypted executable input and the resolved upstream
//! auth. The runtime owns the single bounded retry loop and the durable attempt checkpoint/resolution;
//! this module only revalidates and prepares one attempt at a time.
use std::time::{Duration, Instant};
use async_trait::async_trait;
use crank_core::{
AgentId, ExecutionOrigin, MachineAccessMode, OperationSecurityLevel, PlatformApiKeyScope,
TaskError, TaskLease, TaskOwnerKind, TraceContext, VerifiedMachineCredential, WorkspaceId,
current_unix_ms,
};
use crank_registry::PostgresRegistry;
use crank_registry::PublishedAgentTool;
use crank_runtime::{
ExecutionAuthorization, PrepareAttempt, RuntimeExecutor, RuntimeOperation, SecretCrypto,
TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
TaskAttemptRefusal, TaskHandler, TaskService, TaskWorkerError,
};
use sha2::{Digest, Sha256};
use super::TaskApplication;
use super::auth_resolution::resolve_operation_auth_with;
use super::tool_resolution::runtime_operation;
use crate::access::{allows_scope, credential_allows_security_level};
use crate::catalog::PublishedToolCatalog;
/// Server-owned evidence that one published operation deduplicates repeated sends of the same stable
/// idempotency key.
///
/// The descriptor is constructed by the application for an integration whose upstream contract was
/// actually qualified (key delivery, dedup scope, canonical input binding and a dedup window that
/// covers the pinned retry budget). It is never deserialized from Task input, so a caller key or
/// header alone can never make a write retryable. Production wiring passes an empty set until such a
/// contract exists.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct VerifiedIdempotentWrite {
pub operation_id: String,
pub operation_version: u32,
/// Header through which the stable key is delivered to this exact upstream.
pub key_header: String,
/// Dedup window of the upstream contract, which must cover the pinned retry budget.
pub validity_ms: i64,
}
pub struct TaskWorkerHandler {
registry: PostgresRegistry,
catalog: PublishedToolCatalog,
runtime: RuntimeExecutor,
secret_crypto: SecretCrypto,
service: TaskService,
verified_writes: Vec<VerifiedIdempotentWrite>,
}
impl TaskWorkerHandler {
/// Production construction from the same application entry, registry and runtime the MCP
/// process serves requests with. The worker needs no HTTP state, only these authoritative
/// dependencies.
pub fn from_application(
tasks: &TaskApplication,
registry: PostgresRegistry,
runtime: RuntimeExecutor,
secret_crypto: SecretCrypto,
) -> Self {
Self {
registry,
catalog: tasks.catalog(),
runtime,
secret_crypto,
service: tasks.task_service(),
verified_writes: Vec::new(),
}
}
/// Registers server-owned verified-idempotency evidence of qualified integrations.
pub fn with_verified_writes(mut self, verified_writes: Vec<VerifiedIdempotentWrite>) -> Self {
self.verified_writes = verified_writes;
self
}
/// Authoritative pre-dispatch authorization of one leased Task (Story 4.3).
///
/// Resolves the exact pinned Published Version in the current publication, repeats the approval
/// policy and revalidates the typed persisted Platform API key identity against the registry:
/// the key must still be active, unexpired and bound to this workspace/agent, and its exact
/// scopes/security level must still authorize the operation. Any denial is a typed error, so the
/// worker finishes the attempt without dispatch (fail closed before any network use).
///
/// An administrator Task has no authoritative Task revalidation API in the registry, so it stays
/// fail-closed here instead of inventing authority.
async fn authorize(&self, lease: &TaskLease) -> Result<PublishedAgentTool, TaskError> {
if lease.record.owner_kind != TaskOwnerKind::Agent {
return Err(TaskError::invalid(
"admin task authority revalidation is not implemented",
));
}
let Some(key_id) = lease.record.platform_api_key_id.as_ref() else {
return Err(TaskError::invalid("task authority reference is missing"));
};
let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str());
let agent_id = AgentId::new(lease.record.owner_id.as_str());
let workspace = self
.registry
.get_workspace(&workspace_id)
.await
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?
.ok_or_else(|| TaskError::invalid("task workspace no longer exists"))?;
let agents = self
.registry
.list_agents(&workspace_id)
.await
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?;
let agent = agents
.into_iter()
.find(|agent| agent.id.as_str() == lease.record.owner_id.as_str())
.ok_or_else(|| TaskError::invalid("task agent no longer exists"))?;
let catalog = self
.catalog
.get_catalog(&workspace.workspace.slug, &agent.slug)
.await
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?;
let tool = catalog
.tools
.into_iter()
.find(|tool| {
tool.operation.id.as_str() == lease.record.operation_id.as_str()
&& tool.operation.version == lease.record.pins.operation_version
})
.ok_or_else(|| TaskError::invalid("task published operation pin no longer matches"))?;
if tool.workspace_id.as_str() != lease.record.workspace_id.as_str() {
return Err(TaskError::invalid("task workspace binding mismatch"));
}
if tool.agent_id.as_str() != lease.record.owner_id.as_str() {
return Err(TaskError::invalid("task agent binding mismatch"));
}
if tool
.operation
.execution_config
.approval_policy
.as_ref()
.is_some_and(|policy| policy.required)
{
return Err(TaskError::invalid(
"missing approval proof for task operation",
));
}
// Authoritative revalidation of the exact immutable key identity. A revoked, expired,
// foreign-workspace or foreign-agent key is absent here and never reaches the network.
let key = self
.registry
.revalidate_platform_api_key(&workspace_id, &agent_id, key_id)
.await
.map_err(|_| TaskError::unavailable("task authority store unavailable"))?
.ok_or_else(|| TaskError::invalid("task authority is no longer active"))?;
let credential = VerifiedMachineCredential {
machine_access_mode: MachineAccessMode::StaticAgentKey,
max_security_level: OperationSecurityLevel::Standard,
scopes: key.api_key.scopes.clone(),
platform_api_key_id: Some(key.api_key.id.clone()),
};
if !allows_scope(&credential.scopes, PlatformApiKeyScope::Write) {
return Err(TaskError::invalid("task authority scope is insufficient"));
}
if !credential_allows_security_level(&credential, tool.operation.security_level) {
return Err(TaskError::invalid(
"task authority security level is insufficient",
));
}
Ok(tool)
}
/// Safe-retry classification of one attempt and the stable upstream key digest.
///
/// A read is always safe to repeat. A write is only safe to repeat when this process holds
/// server-owned evidence for the exact published operation and that evidence's dedup window still
/// covers the accepted retry budget; the presence of a client key or an idempotency header alone
/// never qualifies.
fn retry_classification(
&self,
operation: &RuntimeOperation,
input: &serde_json::Value,
retry_window_ms: i64,
) -> (bool, Option<String>) {
if matches!(
&operation.target,
crank_core::Target::Rest(target) if target.method == crank_core::HttpMethod::Get
) {
return (true, None);
}
let Some(policy) = operation.execution_config.idempotency.as_ref() else {
return (false, None);
};
let Some(field) = policy
.input_field
.as_deref()
.filter(|field| !field.trim().is_empty())
else {
return (false, None);
};
let Some(header) = policy
.header_name
.as_deref()
.filter(|value| !value.is_empty())
else {
return (false, None);
};
let key = match input.get(field) {
Some(serde_json::Value::String(value)) if !value.trim().is_empty() => value.clone(),
Some(serde_json::Value::Number(value)) => value.to_string(),
_ => return (false, None),
};
let evidence = self.verified_writes.iter().find(|evidence| {
evidence.operation_id == operation.operation_id.as_str()
&& evidence.operation_version == operation.operation_version
&& evidence.key_header.eq_ignore_ascii_case(header)
&& evidence.validity_ms >= retry_window_ms
});
let Some(evidence) = evidence else {
return (false, None);
};
// Only a digest of the stable key is persisted; the key itself stays in the encrypted input.
let mut digest = Sha256::new();
digest.update(evidence.key_header.as_bytes());
digest.update([0u8]);
digest.update(key.as_bytes());
(true, Some(format!("{:x}", digest.finalize())))
}
/// Story 4.5 producer: the pinned operation still lacks a required input item, so the Task must be
/// suspended before any dispatch. The schema snapshot and its presentation projection are frozen
/// here; the TTL never exceeds the accepted absolute execution deadline.
fn input_request(
&self,
tool: &PublishedAgentTool,
lease: &TaskLease,
missing_path: &str,
expires_at_unix_ms: i64,
) -> Result<crank_core::TaskInputRequest, TaskAttemptRefusal> {
let schema_snapshot = serde_json::to_value(&tool.operation.input_schema).map_err(|_| {
TaskAttemptRefusal::Failed {
code: "task_input_schema_unavailable".to_owned(),
}
})?;
let request = crank_core::TaskInputRequest {
wait_id: format!("wait-{}", uuid::Uuid::now_v7()),
expected_version: lease.record.version.saturating_add(1),
schema_revision: tool.operation.version,
schema_format_version: crank_core::TASK_INPUT_SCHEMA_FORMAT_VERSION,
schema_snapshot,
projected_schema: crate::manifest::schema_to_json_schema(&tool.operation.input_schema),
prompt_code: "required_input_missing".to_owned(),
missing_fields: vec![missing_path.to_owned()],
expires_at_unix_ms,
fingerprint_format_version: crank_core::TASK_INPUT_SCHEMA_FORMAT_VERSION,
};
request.validate().map_err(|_| TaskAttemptRefusal::Failed {
code: "task_input_request_invalid".to_owned(),
})?;
Ok(request)
}
/// Maps a preparation failure onto the attempt refusal vocabulary: a transient authority-store
/// outage leaves the Task leased for recovery, everything else is a proven pre-dispatch refusal.
fn refusal(error: TaskError) -> TaskAttemptRefusal {
match error {
TaskError::Unavailable { .. } => TaskAttemptRefusal::Stopped,
_ => TaskAttemptRefusal::Failed {
code: "task_authority_rejected".to_owned(),
},
}
}
}
#[async_trait]
impl TaskHandler for TaskWorkerHandler {
async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError> {
self.authorize(lease)
.await
.map_err(TaskWorkerError::Store)?;
Ok(())
}
async fn execute(&self, context: TaskAttemptContext) -> TaskAttemptOutcome {
self.runtime.execute_task_outcome(&context, self).await
}
}
#[async_trait]
impl TaskAttemptAuthority for TaskWorkerHandler {
async fn prepare_attempt(
&self,
lease: &TaskLease,
) -> Result<PrepareAttempt, TaskAttemptRefusal> {
let tool = self.authorize(lease).await.map_err(Self::refusal)?;
let input = self
.service
.executable_input(lease)
.await
.map_err(Self::refusal)?;
let operation = runtime_operation(&tool);
let now_ms = current_unix_ms();
let deadline_ms = lease.record.pins.execution_deadline_unix_ms;
if deadline_ms <= now_ms {
// The accepted absolute deadline already elapsed: no further attempt may be dispatched,
// and any previous uncertainty was already recorded as such.
return Err(TaskAttemptRefusal::Failed {
code: "task_deadline_elapsed".to_owned(),
});
}
// Only a missing required item of the pinned schema becomes a typed input request. Any other
// schema violation keeps the existing pre-dispatch attempt validation, and a generation that
// already accepted a continuation can never open a second wait for the same Task.
if let Some(missing_path) =
crank_runtime::missing_required_input(&tool.operation.input_schema, &input)
{
if lease.record.effective_input_fingerprint != lease.record.input_fingerprint {
return Err(TaskAttemptRefusal::Failed {
code: "task_input_still_incomplete".to_owned(),
});
}
return Ok(PrepareAttempt::InputRequired(self.input_request(
&tool,
lease,
&missing_path,
deadline_ms,
)?));
}
let deadline = Instant::now() + Duration::from_millis((deadline_ms - now_ms) as u64);
let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str());
let agent_id = AgentId::new(lease.record.owner_id.as_str());
let resolved_auth = resolve_operation_auth_with(
&self.registry,
&self.secret_crypto,
&workspace_id,
&operation.execution_config,
)
.await
.map_err(|_| TaskAttemptRefusal::Failed {
code: "task_auth_resolution_failed".to_owned(),
})?;
// Continue the parent trace persisted with the accepted Task; every attempt adds its own
// child `adapter_attempt` span carrying the committed durable attempt ID.
let trace_context =
TraceContext::parse(&lease.record.support.traceparent).map_err(|_| {
TaskAttemptRefusal::Failed {
code: "task_trace_parent_invalid".to_owned(),
}
})?;
let request_context = crank_runtime::RuntimeRequestContext::new(
lease.record.support.request_id.clone(),
trace_context,
)
.with_metering_context(
workspace_id.clone(),
Some(agent_id.clone()),
crank_core::InvocationSource::AgentToolCall,
);
let (retry_allowed, upstream_key_digest) =
self.retry_classification(&operation, &input, deadline_ms - now_ms);
Ok(PrepareAttempt::Ready(Box::new(TaskAttemptPlan {
workspace_id,
origin: ExecutionOrigin::AgentSnapshot,
agent_id: Some(agent_id),
operation,
input,
authorization: ExecutionAuthorization::Authorized,
resolved_auth,
request_context,
deadline,
retry_allowed,
upstream_key_digest,
})))
}
}
+121 -2
View File
@@ -14,8 +14,8 @@ use std::sync::Arc;
use std::time::{Duration, Instant};
use crank_core::{
CorrelationContext, TaskAcceptance, TaskAccess, TaskError, TaskId, TaskOwnerKind, TaskRecord,
ToolExecutionMode,
CorrelationContext, PlatformApiKeyId, TaskAcceptance, TaskAccess, TaskError, TaskId,
TaskOwnerKind, TaskRecord, ToolExecutionMode,
};
use crank_registry::{PostgresRegistry, PublishedAgentTool};
use crank_runtime::{TaskAcceptDraft, TaskOutcome, TaskService};
@@ -64,6 +64,20 @@ impl TaskPrincipal {
}
}
/// Typed immutable Platform API key identity of this principal. A verified agent credential
/// without a key ID cannot be revalidated by a worker, so it is refused instead of accepted;
/// an administrator has no machine key and is stored as `None` (the worker fails closed).
fn platform_api_key_id(&self) -> Result<Option<PlatformApiKeyId>, TaskApplicationError> {
match self {
Self::Agent { credential, .. } => credential
.platform_api_key_id
.clone()
.map(Some)
.ok_or(TaskApplicationError::Denied),
Self::Admin { .. } => Ok(None),
}
}
/// Bounded server-derived authorization scope identity of this principal.
fn authorization_scope(&self) -> String {
let identity = match self {
@@ -131,6 +145,17 @@ impl TaskApplication {
}
}
/// Runtime task service carrying encrypted executable input.
pub fn task_service(&self) -> TaskService {
self.service.clone()
}
/// Published catalog of the same application entry, for the durable Task worker's pre-dispatch
/// revalidation.
pub(super) fn catalog(&self) -> PublishedToolCatalog {
self.catalog.clone()
}
/// Readiness of the durable Task store this entry depends on.
pub async fn health(&self) -> Result<(), TaskError> {
self.service.health().await
@@ -170,6 +195,7 @@ impl TaskApplication {
// of accepting work a worker could never safely execute.
return Err(TaskApplicationError::ApprovalRequired);
}
let platform_api_key_id = principal.platform_api_key_id()?;
let draft = |operation_version: u32| {
let (owner_kind, owner_id) = principal.owner(&tool);
TaskAcceptDraft {
@@ -177,6 +203,7 @@ impl TaskApplication {
owner_kind,
owner_id,
authorization_scope: principal.authorization_scope(),
platform_api_key_id: platform_api_key_id.clone(),
operation_id: tool.operation.id.as_str().to_owned(),
operation_version,
policy_mode: policy.mode,
@@ -186,6 +213,7 @@ impl TaskApplication {
deadline: Instant::now() + Duration::from_millis(policy.deadline_ms.max(1)),
retention_ms: TaskService::default_retention_ms(),
approval_reference: None,
retry: task_retry_policy(&tool),
}
};
@@ -247,6 +275,82 @@ impl TaskApplication {
}
}
/// Owner-scoped cancellation of one Task of the addressed published Tool (Story 4.4).
///
/// Cancellation is decided by the durable store under a row lock: before the dispatch checkpoint
/// it is an honest cancellation with zero outbound calls, after it the Task becomes the
/// non-terminal uncertain outcome instead of a plain cancel or a retry.
pub async fn cancel(
&self,
principal: &TaskPrincipal,
tool_name: &str,
task_id: TaskId,
expected_version: Option<u32>,
) -> Result<crank_core::TaskCancelOutcome, TaskApplicationError> {
let access = self.task_access(principal, tool_name).await?;
match self
.service
.cancel(&access, task_id, expected_version, "user_cancelled")
.await
{
Ok(outcome) => Ok(outcome),
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
Err(error) => Err(TaskApplicationError::Task(error)),
}
}
/// Applies one additional validated input to a waiting Task (Story 4.5).
///
/// This is the server-internal application entry: the public Task MCP API stays Story 4.7. The
/// principal is already authenticated here, the owner scope is derived from the verified
/// credential and the durable transaction revalidates the same key before any receipt lookup.
/// An identical authorized replay returns the committed outcome; a conflicting or stale answer is
/// typed, and an elapsed TTL never revives the Task.
#[allow(clippy::too_many_arguments)]
pub async fn update_input(
&self,
principal: &TaskPrincipal,
tool_name: &str,
task_id: TaskId,
wait_id: &str,
expected_version: u32,
update_key: &str,
input: &Value,
) -> Result<crank_core::TaskInputUpdateOutcome, TaskApplicationError> {
let access = self.task_access(principal, tool_name).await?;
match self
.service
.update_input(
&access,
task_id,
wait_id,
expected_version,
update_key,
input,
)
.await
{
Ok(outcome) => Ok(outcome),
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
Err(error) => Err(TaskApplicationError::Task(error)),
}
}
/// Owner-scoped attempt history of one Task of the addressed published Tool.
pub async fn attempts(
&self,
principal: &TaskPrincipal,
tool_name: &str,
task_id: TaskId,
) -> Result<Vec<crank_core::TaskAttempt>, TaskApplicationError> {
let access = self.task_access(principal, tool_name).await?;
match self.service.attempts(&access, task_id).await {
Ok(attempts) => Ok(attempts),
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
Err(error) => Err(TaskApplicationError::Task(error)),
}
}
async fn task_access(
&self,
principal: &TaskPrincipal,
@@ -293,3 +397,18 @@ impl TaskApplication {
Ok(tool)
}
}
/// Bounded retry snapshot pinned from the exact published operation at acceptance. A later
/// publication, a restart or a new binary never changes the numbers an accepted Task was admitted
/// with; a missing or oversized policy is clamped to the closed story bound.
fn task_retry_policy(tool: &PublishedAgentTool) -> crank_core::TaskRetryPolicy {
let max_attempts = tool
.operation
.execution_config
.retry_policy
.as_ref()
.map_or(1, |policy| {
policy.max_attempts.clamp(1, crank_core::TASK_MAX_ATTEMPTS)
});
crank_core::TaskRetryPolicy::new(max_attempts, 1)
}
@@ -415,7 +415,7 @@ pub(super) fn agent_id() -> AgentId {
AgentId::new(format!("agent_{AGENT_SLUG}"))
}
pub(super) async fn publish_agent(
pub(crate) async fn publish_agent(
registry: &PostgresRegistry,
operation: &Operation<Schema, MappingSet>,
) {
@@ -470,7 +470,7 @@ pub(super) async fn publish_agent(
.expect("the agent must be published");
}
pub(super) async fn create_api_key(registry: &PostgresRegistry) -> String {
pub(crate) async fn create_api_key(registry: &PostgresRegistry) -> String {
let secret = format!("crk_dispatch_observer_{}", uuid::Uuid::now_v7().simple());
let api_key = PlatformApiKey {
id: PlatformApiKeyId::new("pk_dispatch_observer"),
@@ -500,7 +500,7 @@ pub(super) fn hash_access_secret(secret: &str) -> String {
URL_SAFE_NO_PAD.encode(Sha256::digest(secret.as_bytes()))
}
pub(super) fn test_operation(base_url: &str) -> Operation<Schema, MappingSet> {
pub(crate) fn test_operation(base_url: &str) -> Operation<Schema, MappingSet> {
Operation {
id: OperationId::new(format!("op_{TOOL_NAME}")),
name: TOOL_NAME.to_owned(),
+2 -1
View File
@@ -16,7 +16,8 @@ mod tool_search;
mod transport;
pub use app::{
TaskApplication, TaskApplicationError, TaskPrincipal, build_app, build_app_with_artifacts,
TaskApplication, TaskApplicationError, TaskPrincipal, TaskWorkerHandler,
VerifiedIdempotentWrite, build_app, build_app_with_artifacts,
build_app_with_artifacts_and_tasks, build_app_with_background_workers,
build_app_with_background_workers_and_limits,
};
@@ -41,6 +41,9 @@ use std::{
sync::{Mutex, MutexGuard},
};
// Both pinned revisions still require logging despite the SDK's SEP-2577 deprecation.
#[allow(deprecated)]
use rmcp::model::{LoggingLevel, LoggingMessageNotificationParam, SetLevelRequestParams};
use serde::Deserialize;
use serde_json::{Value, json};
use tokio::task_local;
@@ -61,7 +64,8 @@ pub(crate) enum LogLevel {
}
impl LogLevel {
/// The exact wire literal of this level.
/// The exact wire literal of this level (contract assertions only; runtime uses SDK serde).
#[cfg(test)]
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Debug => "debug",
@@ -80,17 +84,47 @@ impl LogLevel {
/// Nothing is coerced: no case folding, no aliases (`warn`), no numeric severities. A value that
/// is not one of the pinned strings is malformed, which is what the pinned `LoggingLevel` type
/// says and what `InvalidParamsError` is for.
#[allow(deprecated)]
pub(crate) fn parse(value: &str) -> Option<Self> {
match value {
"debug" => Some(Self::Debug),
"info" => Some(Self::Info),
"notice" => Some(Self::Notice),
"warning" => Some(Self::Warning),
"error" => Some(Self::Error),
"critical" => Some(Self::Critical),
"alert" => Some(Self::Alert),
"emergency" => Some(Self::Emergency),
_ => None,
if value.len() > MAX_LEVEL_BYTES {
return None;
}
let level: LoggingLevel = serde_json::from_value(Value::String(value.to_owned())).ok()?;
#[cfg(test)]
crate::app::dispatch_observer::sdk_logging_level_parsed();
Some(level.into())
}
}
// Keep domain severity ordering independent of SDK enum discriminants (which have no Ord).
#[allow(deprecated)]
impl From<LoggingLevel> for LogLevel {
fn from(level: LoggingLevel) -> Self {
match level {
LoggingLevel::Debug => Self::Debug,
LoggingLevel::Info => Self::Info,
LoggingLevel::Notice => Self::Notice,
LoggingLevel::Warning => Self::Warning,
LoggingLevel::Error => Self::Error,
LoggingLevel::Critical => Self::Critical,
LoggingLevel::Alert => Self::Alert,
LoggingLevel::Emergency => Self::Emergency,
}
}
}
#[allow(deprecated)]
impl From<LogLevel> for LoggingLevel {
fn from(level: LogLevel) -> Self {
match level {
LogLevel::Debug => Self::Debug,
LogLevel::Info => Self::Info,
LogLevel::Notice => Self::Notice,
LogLevel::Warning => Self::Warning,
LogLevel::Error => Self::Error,
LogLevel::Critical => Self::Critical,
LogLevel::Alert => Self::Alert,
LogLevel::Emergency => Self::Emergency,
}
}
}
@@ -135,11 +169,20 @@ pub(crate) struct SetLevelCommand {
impl SetLevelCommand {
/// The bounded input contract of AC2: the level is length-bounded and compared exactly, so a
/// malformed or oversized value is refused with bounded work and one safe detail.
pub(crate) fn validate(&self) -> Result<LogLevel, &'static str> {
#[allow(deprecated)]
pub(crate) fn validate(&self) -> Result<SetLevelRequestParams, &'static str> {
if self.level.is_empty() || self.level.len() > MAX_LEVEL_BYTES {
return Err(INVALID_LEVEL);
}
LogLevel::parse(&self.level).ok_or(INVALID_LEVEL)
// Only level was part of the old command: unknown members (including _meta) stay ignored.
let request =
serde_json::from_value(json!({"level": self.level})).map_err(|_| INVALID_LEVEL)?;
#[cfg(test)]
{
crate::app::dispatch_observer::sdk_logging_level_parsed();
crate::app::dispatch_observer::sdk_logging_dispatch_enter();
}
Ok(request)
}
}
@@ -282,19 +325,22 @@ pub(crate) fn take_notifications() -> Vec<Value> {
}
std::mem::take(&mut log.pending)
.into_iter()
.map(|event| {
json!({
.filter_map(|event| {
#[allow(deprecated)]
let params = LoggingMessageNotificationParam::new(
event.level.into(),
json!({"event": event.event, "outcome": event.outcome}),
)
.with_logger(LOGGER_NAME);
// Logging remains best-effort: a serialization failure drops only this event.
let params = serde_json::to_value(params).ok()?;
#[cfg(test)]
crate::app::dispatch_observer::sdk_logging_notification_serialized();
Some(json!({
"jsonrpc": "2.0",
"method": "notifications/message",
"params": {
"level": event.level.as_str(),
"logger": LOGGER_NAME,
"data": {
"event": event.event,
"outcome": event.outcome,
}
}
})
"params": params,
}))
})
.collect()
})
@@ -354,7 +400,9 @@ pub(crate) fn handle_logging_command(
command: SetLevelCommand,
) -> LoggingOutcome {
match command.validate() {
Ok(level) => {
Ok(request) => {
#[allow(deprecated)]
let level = request.level.into();
store.set(session_id, level);
LoggingOutcome::Applied(level)
}
@@ -10,7 +10,8 @@ use super::*;
/// is malformed (which the adapter answers with the same safe detail).
fn validate(level: &str) -> Option<Result<LogLevel, &'static str>> {
let command: SetLevelCommand = serde_json::from_value(json!({ "level": level })).ok()?;
Some(command.validate())
#[allow(deprecated)]
Some(command.validate().map(|request| request.level.into()))
}
#[test]
@@ -291,8 +292,11 @@ fn the_modern_level_is_read_from_the_pinned_meta_key_only() {
fn the_command_records_the_level_of_its_own_session_only() {
let store = LogLevelStore::new();
store.set("other-session", LogLevel::Debug);
let command: SetLevelCommand =
serde_json::from_value(json!({ "level": "critical" })).expect("a pinned level parses");
// Unknown members were ignored by the legacy command; SDK metadata must not narrow that wire.
let command: SetLevelCommand = serde_json::from_value(json!({
"level": "critical", "_meta": 7, "unknown": {"private": "ignored"}
}))
.expect("a pinned level parses regardless of unknown members");
let outcome = handle_logging_command(&store, "this-session", command);
assert!(matches!(
outcome,
@@ -36,7 +36,10 @@ pub(crate) use command::{
ContinuationSupport, ServiceContext, ToolCallCommand, ToolCallOutcome, ToolsCommand,
ToolsListCommand, ToolsOutcome, handle_tools_command,
};
pub(crate) use completion::{CompletionCommand, CompletionOutcome, handle_completion_command};
pub(crate) use completion::{
CompletionArgument, CompletionCommand, CompletionOutcome, CompletionReference,
handle_completion_command,
};
pub(crate) use logging::{
CarriedScope, EVENT_FAILED, EVENT_LEVEL_APPLIED, EVENT_REJECTED, EVENT_SERVED, INVALID_LEVEL,
LogLevel, LogLevelStore, LoggingOutcome, SetLevelCommand, carry_scope, handle_logging_command,
@@ -0,0 +1,345 @@
//! #107 Completion: real build_app HTTP wire controls and a separate SDK-path oracle.
use super::{CounterSet, observe, support::*};
use axum::{body::Body, response::Response};
use futures_util::{StreamExt, stream};
use serde_json::{Value, json};
use std::{sync::Arc, time::Duration};
#[path = "rmcp_completion_test_support.rs"]
mod fixtures;
use fixtures::{BOUNDED_URI, Endpoint, LARGE_URI, URI};
fn observe_body(response: Response, counters: Arc<CounterSet>) -> Response {
let (parts, body) = response.into_parts();
let body = stream::unfold(
(body.into_data_stream(), counters),
|(mut body, c)| async move {
observe(c.clone(), body.next())
.await
.map(|chunk| (chunk, (body, c)))
},
);
Response::from_parts(parts, Body::from_stream(body))
}
fn counts(c: &CounterSet) -> [usize; 2] {
[c.sdk_completion_dispatch(), c.sdk_completion_result()]
}
fn params(reference: Value, name: &str, value: &str) -> Value {
json!({"ref":reference,"argument":{"name":name,"value":value}})
}
fn prompt(name: &str) -> Value {
json!({"type":"ref/prompt","name":name})
}
fn resource(uri: &str) -> Value {
json!({"type":"ref/resource","uri":uri})
}
fn request(revision: &str, mut params: Value) -> Value {
if revision == MODERN_REVISION {
params["_meta"] = modern_meta();
}
json!({"jsonrpc":"2.0","id":9007199254740993_u64,"method":"completion/complete","params":params})
}
async fn wire(response: reqwest::Response, sse: bool) -> Value {
assert_eq!(response.status(), 200);
assert_eq!(response.headers()["x-request-id"], "sdk-completion-wire");
assert_eq!(
response.headers()["x-trace-id"],
"11111111111111111111111111111111"
);
let content_type = response.headers()["content-type"]
.to_str()
.unwrap()
.to_owned();
let text = tokio::time::timeout(Duration::from_secs(5), response.text())
.await
.unwrap()
.unwrap();
let body: Value = if sse {
assert!(
content_type.starts_with("text/event-stream"),
"{content_type}: {text}"
);
let frames: Vec<Value> = text
.lines()
.filter_map(|line| {
line.strip_prefix("data:")
.map(|s| serde_json::from_str(s.trim()).unwrap())
})
.collect();
let terminal: Vec<_> = frames
.iter()
.filter(|f| f.get("id") == Some(&json!(9007199254740993_u64)))
.collect();
assert_eq!(terminal.len(), 1, "{frames:?}");
assert_eq!(frames.last(), Some(terminal[0]), "terminal must be last");
terminal[0].clone()
} else {
assert!(
content_type.starts_with("application/json"),
"{content_type}: {text}"
);
serde_json::from_str(&text).unwrap()
};
assert_eq!(body["jsonrpc"], "2.0");
assert_eq!(body["id"], json!(9007199254740993_u64));
body
}
async fn success(
h: &Endpoint,
revision: &str,
session: Option<&str>,
p: Value,
sse: bool,
) -> (Value, [usize; 2]) {
let before = counts(&h.counters);
let body = wire(
h.post(revision, session, &request(revision, p), sse).await,
sse,
)
.await;
assert!(body.get("error").is_none(), "{body}");
let result = &body["result"];
if revision == MODERN_REVISION {
assert_eq!(result["resultType"], "complete", "{body}");
} else {
assert!(result.get("resultType").is_none(), "{body}");
}
for key in ["ttlMs", "cacheScope", "_meta"] {
assert!(result.get(key).is_none(), "{body}");
}
let after = counts(&h.counters);
(
result["completion"].clone(),
std::array::from_fn(|i| after[i] - before[i]),
)
}
async fn control(revision: &str) -> Vec<[usize; 2]> {
let h = Endpoint::started().await;
let session = h.session(revision).await;
let session = session.as_deref();
let modes: &[bool] = if revision == MODERN_REVISION {
&[false, true]
} else {
&[false]
};
let mut deltas = Vec::new();
for &sse in modes {
for (p, expected) in [
(
params(prompt("greeting"), "tone", ""),
json!({"values":["warm","dry"],"total":2}),
),
(
params(prompt("archived-prompt"), "tone", "W"),
json!({"values":["warm"],"total":1}),
),
(
params(prompt("greeting"), "name", "A"),
json!({"values":[],"total":0}),
),
(
params(resource(URI), "region", ""),
json!({"values":["eu","us","apac"],"total":3}),
),
(
params(resource(URI), "region", "A"),
json!({"values":["apac"],"total":1}),
),
(
params(resource(URI), "region", "zzz"),
json!({"values":[],"total":0}),
),
] {
let (completion, delta) = success(&h, revision, session, p, sse).await;
assert_eq!(completion, expected);
deltas.push(delta);
}
for (uri, total) in [(BOUNDED_URI, Some(150)), (LARGE_URI, None)] {
let (completion, delta) = success(
&h,
revision,
session,
params(resource(uri), "region", "item"),
sse,
)
.await;
assert_eq!(
completion["values"],
json!((0..100).map(|i| format!("item{i:04}")).collect::<Vec<_>>())
);
assert_eq!(completion["hasMore"], true);
if let Some(total) = total {
assert_eq!(completion["total"], total);
} else {
assert!(
completion.get("total").is_none(),
"partial scan: {completion}"
);
}
deltas.push(delta);
}
let mut with_context = params(prompt("greeting"), "tone", "w");
with_context["context"] = json!({"arguments":{"name":"Ada","tone":"dry"}});
let (completion, delta) = success(&h, revision, session, with_context, sse).await;
assert_eq!(completion, json!({"values":["warm"],"total":1}));
deltas.push(delta);
let mut unavailable = None;
for p in [
params(prompt("draft-prompt"), "tone", ""),
params(prompt("unbound-prompt"), "tone", ""),
params(prompt("other-agent-prompt"), "tone", ""),
params(prompt("never-existed"), "tone", ""),
params(prompt("changed-draft"), "tone", ""),
params(prompt("greeting"), "unknown-argument", ""),
params(resource("crank://foreign/{region}"), "region", ""),
params(resource(URI), "unknown-parameter", ""),
] {
let body = wire(
h.post(revision, session, &request(revision, p), sse).await,
sse,
)
.await;
assert_eq!(body["error"]["code"], -32602, "{body}");
assert_eq!(
body["error"]["message"],
"the completion reference is not available"
);
assert!(body.get("result").is_none());
if let Some(ref previous) = unavailable {
assert_eq!(&body, previous);
}
unavailable = Some(body);
}
let mut invalid = vec![
json!({"argument":{"name":"tone","value":""}}),
params(json!({"type":"ref/tool","name":"greeting"}), "tone", ""),
params(prompt("greeting"), "tone", &"s".repeat(1025)),
params(prompt("greeting"), &"n".repeat(129), ""),
];
// Preserve domain validation on arbitrary JSON, never drop or stringify context values for SDK.
for value in [
json!(7),
json!(true),
json!(null),
json!(["eu"]),
json!({"region":"eu"}),
json!("s".repeat(1025)),
] {
let mut p = params(prompt("greeting"), "tone", "");
p["context"] = json!({"arguments":{"secret-context":value}});
invalid.push(p);
}
let mut p = params(prompt("greeting"), "tone", "");
p["context"] = json!({"arguments":(0..33).map(|i| (format!("arg{i}"), json!("eu"))).collect::<serde_json::Map<_,_>>()});
invalid.push(p);
for p in invalid {
let body = wire(
h.post(revision, session, &request(revision, p), sse).await,
sse,
)
.await;
assert_eq!(body["error"]["code"], -32602, "{body}");
assert_eq!(
body["error"]["message"],
"invalid completion/complete parameters"
);
assert!(body.get("result").is_none());
for leaked in ["secret-context", &"s".repeat(1025), &"n".repeat(129)] {
assert!(!body.to_string().contains(leaked), "{body}");
}
}
}
h.registry
.revoke_platform_api_key(
&workspace_id(),
&crank_core::PlatformApiKeyId::new("pk_dispatch_observer"),
&time::OffsetDateTime::now_utc(),
)
.await
.unwrap();
let before = h.counters.snapshot();
let sdk_before = counts(&h.counters);
let response = h
.post(
revision,
session,
&request(revision, params(prompt("greeting"), "tone", "")),
false,
)
.await;
assert_eq!(response.status(), 401);
let _ = response.bytes().await.unwrap();
assert_eq!(
counts(&h.counters),
sdk_before,
"revoked key must not dispatch to SDK"
);
let after = h.counters.snapshot();
assert_eq!(after.era_modern, before.era_modern);
assert_eq!(after.era_legacy, before.era_legacy);
assert_eq!(after.tools_service, before.tools_service);
eprintln!(
"existing GREEN Completion HTTP {revision}: Prompt+Resource refs, exact bound v1/archive, prefix/empty/bounded/truncated, JSON/context/errors/correlation/auth; SDK actual={deltas:?}, expected=[1,1] per success"
);
deltas
}
#[tokio::test]
async fn rmcp_completion_wire_legacy() {
control(LEGACY_REVISION).await;
}
#[tokio::test]
async fn rmcp_completion_wire_modern() {
control(MODERN_REVISION).await;
}
#[tokio::test]
async fn rmcp_completion_sdk_legacy() {
let deltas = control(LEGACY_REVISION).await;
assert!(
deltas.iter().all(|d| *d == [1, 1]),
"SDK typed dispatch/result missing after successful HTTP: {deltas:?}"
);
}
#[tokio::test]
async fn rmcp_completion_sdk_modern() {
let deltas = control(MODERN_REVISION).await;
assert!(
deltas.iter().all(|d| *d == [1, 1]),
"SDK typed dispatch/result missing after successful HTTP: {deltas:?}"
);
}
#[tokio::test]
async fn rmcp_completion_observer_scope_control() {
use super::{sdk_completion_dispatch_enter, sdk_completion_result_serialized};
let a = Arc::new(CounterSet::default());
let b = Arc::new(CounterSet::default());
sdk_completion_dispatch_enter();
sdk_completion_result_serialized();
assert_eq!(counts(&a), [0; 2]);
let snapshot = a.snapshot();
tokio::join!(
observe(a.clone(), async {
sdk_completion_dispatch_enter();
tokio::task::yield_now().await;
sdk_completion_result_serialized();
}),
observe(b.clone(), async {
sdk_completion_result_serialized();
})
);
assert_eq!(counts(&a), [1, 1]);
assert_eq!(counts(&b), [0, 1]);
assert_eq!(a.snapshot(), snapshot);
let body = Body::from_stream(stream::once(async {
sdk_completion_result_serialized();
Ok::<_, std::convert::Infallible>("data: terminal\n\n")
}));
let response = observe_body(Response::new(body), b.clone());
assert_eq!(counts(&b), [0, 1], "body must be deferred");
assert_eq!(
&axum::body::to_bytes(response.into_body(), 100)
.await
.unwrap()[..],
b"data: terminal\n\n"
);
assert_eq!(counts(&b), [0, 2]);
assert_eq!(counts(&a), [1, 1]);
}
@@ -0,0 +1,445 @@
//! Completion HTTP fixture using the existing observed HTTP app and disposable PostgreSQL.
use super::*;
use crank_core::prompt::{PromptDefinition, PromptSensitivity};
use crank_core::{
Agent, AgentStatus, AgentVersion, HttpMethod, RestTarget, Target, ToolSelectionPolicy,
ids::PromptId,
};
use crank_registry::PostgresRegistry;
use crank_registry::model::{
AgentPromptBinding, ArchivePromptRequest, CreatePromptDraftRequest, PublishPromptRequest,
SaveAgentCatalogConfigRequest, UpdatePromptDraftRequest,
};
use crank_registry::{CreateAgentRequest, PublishAgentRequest};
use crank_runtime::{
InMemoryCoordinationStateStore, RequestRateLimitConfig, RequestRateLimiter, SecretCrypto,
};
pub(super) fn definition(name: &str, content: &str) -> PromptDefinition {
PromptDefinition::parse(&json!({
"name":name, "description":format!("Published {name}"),
"arguments":[
{"name":"name","schema":{"type":"string"},"required":true},
{"name":"count","schema":{"type":"integer"},"required":false,"default":2},
{"name":"enabled","schema":{"type":"boolean"},"required":false,"default":true},
{"name":"tone","schema":{"type":"string","enum":["warm","dry"]},"required":false,"default":"warm"}
],
"messages":[{"role":"system","content":"Be brief."},{"role":"user","content":content}]
})).unwrap()
}
pub(super) struct Endpoint {
pub registry: crank_registry::PostgresRegistry,
pub counters: Arc<CounterSet>,
pub key: String,
pub url: String,
server: tokio::task::JoinHandle<()>,
}
impl Drop for Endpoint {
fn drop(&mut self) {
self.server.abort();
}
}
impl Endpoint {
pub async fn started() -> Self {
crank_test_support::postgres_schema_url_for_task("completion_owner", "t_3fa1ad63").await;
let registry = test_registry().await;
publish_region_operation(&registry, "completion_lookup").await;
for (id, uri, size) in [
("res_completion_small", URI, 3),
("res_completion_large", LARGE_URI, 1025),
("res_completion_bounded", BOUNDED_URI, 150),
] {
publish_template(&registry, &ResourceId::new(id), uri, size).await;
}
let mut archived_at = test_time();
for (id, name, publish) in [
("prm_a", "greeting", true),
("prm_b", "archived-prompt", true),
("prm_c", "unbound-prompt", true),
("prm_d", "draft-prompt", false),
("prm_e", "other-agent-prompt", true),
] {
let row = registry
.create_prompt_draft(CreatePromptDraftRequest {
workspace_id: workspace_id(),
prompt_id: PromptId::new(id),
sensitivity: PromptSensitivity::Internal,
definition: definition(
name,
"Hello {{name}} count={{count}} enabled={{enabled}} tone={{tone}}",
),
created_at: time::OffsetDateTime::now_utc(),
})
.await
.unwrap();
if publish {
let row = registry
.publish_prompt_draft(PublishPromptRequest {
workspace_id: workspace_id(),
prompt_id: PromptId::new(id),
expected_updated_at: row.updated_at,
published_at: time::OffsetDateTime::now_utc(),
})
.await
.unwrap();
if id == "prm_b" {
archived_at = row.updated_at;
}
}
}
for (id, slug) in [
(agent_id(), AGENT_SLUG),
(
crank_core::AgentId::new("agent_other_prompts"),
"other-prompts",
),
] {
registry
.create_agent(CreateAgentRequest {
agent: &Agent {
id: id.clone(),
workspace_id: workspace_id(),
slug: slug.into(),
display_name: "SDK Prompts".into(),
description: String::new(),
status: AgentStatus::Draft,
current_draft_version: 1,
latest_published_version: None,
created_at: test_time(),
updated_at: test_time(),
published_at: None,
},
version: &AgentVersion {
agent_id: id,
version: 1,
status: AgentStatus::Draft,
instructions: json!({}),
tool_selection_policy: ToolSelectionPolicy::default(),
created_at: test_time(),
},
bindings: &[],
})
.await
.unwrap();
}
let other_agent = crank_core::AgentId::new("agent_other_prompts");
registry
.save_agent_catalog_config_editable_with_prompts(
SaveAgentCatalogConfigRequest {
workspace_id: &workspace_id(),
agent_id: &other_agent,
agent_version: 1,
bindings: &[],
tool_selection_policy: &ToolSelectionPolicy::default(),
resource_bindings: &[],
expected_state: None,
},
&[AgentPromptBinding {
prompt_id: PromptId::new("prm_e"),
prompt_version: 1,
}],
)
.await
.unwrap();
registry
.publish_agent(PublishAgentRequest {
workspace_id: &workspace_id(),
agent_id: &other_agent,
version: 1,
published_at: &test_time(),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
registry
.save_agent_catalog_config_editable_with_prompts(
SaveAgentCatalogConfigRequest {
workspace_id: &workspace_id(),
agent_id: &agent_id(),
agent_version: 1,
bindings: &[],
tool_selection_policy: &ToolSelectionPolicy::default(),
resource_bindings: &[
AgentResourceBinding {
resource_id: ResourceId::new("res_completion_small"),
resource_version: 1,
},
AgentResourceBinding {
resource_id: ResourceId::new("res_completion_large"),
resource_version: 1,
},
AgentResourceBinding {
resource_id: ResourceId::new("res_completion_bounded"),
resource_version: 1,
},
],
expected_state: None,
},
&[
AgentPromptBinding {
prompt_id: PromptId::new("prm_a"),
prompt_version: 1,
},
AgentPromptBinding {
prompt_id: PromptId::new("prm_b"),
prompt_version: 1,
},
],
)
.await
.unwrap();
registry
.publish_agent(PublishAgentRequest {
workspace_id: &workspace_id(),
agent_id: &agent_id(),
version: 1,
published_at: &test_time(),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
registry
.archive_prompt_draft(ArchivePromptRequest {
workspace_id: workspace_id(),
prompt_id: PromptId::new("prm_b"),
expected_updated_at: archived_at,
archived_at: time::OffsetDateTime::now_utc(),
})
.await
.unwrap();
// A mutable v2 draft must not change the bound v1 rendering or discovery metadata.
let draft = registry
.get_prompt_draft(&workspace_id(), &PromptId::new("prm_a"))
.await
.unwrap();
registry
.update_prompt_draft(UpdatePromptDraftRequest {
workspace_id: workspace_id(),
prompt_id: PromptId::new("prm_a"),
sensitivity: PromptSensitivity::Internal,
definition: definition("changed-draft", "Changed {{name}}"),
expected_updated_at: draft.updated_at,
updated_at: time::OffsetDateTime::now_utc(),
})
.await
.unwrap();
let key = create_api_key(&registry).await;
let counters = Arc::new(CounterSet::default());
let app = crate::build_app(
registry.clone(),
Duration::ZERO,
None,
SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(),
crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build(),
RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()),
Arc::new(InMemoryCoordinationStateStore::default()),
Arc::new(crate::session::InMemorySessionStore::default()),
Arc::new(crate::auth::CommunityMachineCredentialVerifier),
);
let app = observe_requests(app, counters.clone(), None).layer(axum::middleware::from_fn({
let counters = counters.clone();
move |request: axum::extract::Request, next: axum::middleware::Next| {
let counters = counters.clone();
async move { observe_body(next.run(request).await, counters) }
}
}));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!(
"http://{}/v1/{WORKSPACE_SLUG}/{AGENT_SLUG}",
listener.local_addr().unwrap()
);
let server = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
Self {
registry,
counters,
key,
url,
server,
}
}
pub async fn post(
&self,
revision: &str,
session: Option<&str>,
body: &Value,
sse: bool,
) -> reqwest::Response {
let mut request = reqwest::Client::new()
.post(&self.url)
.bearer_auth(&self.key)
.header(
"Accept",
if sse {
"text/event-stream, application/json"
} else {
"application/json"
},
)
.header("x-request-id", "sdk-completion-wire")
.header(
"traceparent",
"00-11111111111111111111111111111111-2222222222222222-01",
)
.header("MCP-Protocol-Version", revision);
if let Some(session) = session {
request = request.header("MCP-Session-Id", session);
}
if revision == MODERN_REVISION {
request = request.header("Mcp-Method", body["method"].as_str().unwrap());
}
request.json(body).send().await.unwrap()
}
pub async fn session(&self, revision: &str) -> Option<String> {
if revision == MODERN_REVISION {
return None;
}
let response = self
.post(revision, None, &legacy_initialize(1), false)
.await;
assert_eq!(response.status(), 200);
let session = response.headers()["MCP-Session-Id"]
.to_str()
.unwrap()
.to_owned();
assert_eq!(
json_body(response).await["result"]["protocolVersion"],
revision
);
assert_eq!(
self.post(revision, Some(&session), &initialized_notification(), false)
.await
.status(),
202
);
Some(session)
}
}
use crank_core::{ids::ResourceId, resource::TemplateSource};
use crank_mapping::{MappingRule, MappingSet};
use crank_registry::model::{
CreateResourceTemplateDraftRequest, PublishResourceTemplateVersionRequest,
};
use crank_registry::{AgentResourceBinding, PublishRequest};
use sha2::{Digest, Sha256};
pub(super) const URI: &str = "crank://completion-orders/{region}";
pub(super) const LARGE_URI: &str = "crank://completion-large/{region}";
pub(super) const BOUNDED_URI: &str = "crank://completion-bounded/{region}";
async fn publish_region_operation(registry: &PostgresRegistry, name: &str) {
let mut operation = test_operation("http://127.0.0.1:9");
operation.id = crank_core::OperationId::new(format!("op_{name}"));
operation.name = name.to_owned();
operation.display_name = "Region lookup".to_owned();
operation.target = Target::Rest(RestTarget {
base_url: "http://127.0.0.1:9".to_owned(),
method: HttpMethod::Get,
path_template: "/regions/{region}".to_owned(),
static_headers: Default::default(),
request_contract: None,
response_contract: Vec::new(),
multipart_field_mime_types: Default::default(),
});
operation.input_schema = object_schema("region");
operation
.input_schema
.fields
.get_mut("region")
.unwrap()
.enum_values = vec!["eu".to_owned(), "us".to_owned(), "apac".to_owned()];
operation.input_mapping = MappingSet {
rules: vec![MappingRule {
source: "$.mcp.region".to_owned(),
target: "$.request.path.region".to_owned(),
required: true,
default_value: None,
transform: None,
condition: None,
notes: None,
}],
};
registry
.create_operation(&workspace_id(), &operation, Some("alice"))
.await
.expect("the Read Operation is created through the registry API");
registry
.publish_operation(PublishRequest {
workspace_id: &workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &time::OffsetDateTime::now_utc(),
published_by: Some("alice"),
})
.await
.expect("the Read Operation exact version publishes");
}
/// One published Resource template with an enumerated `region` parameter.
async fn publish_template(
registry: &PostgresRegistry,
resource_id: &ResourceId,
uri_template: &str,
size: usize,
) {
let operation_name = "completion_lookup";
let options: Vec<String> = if size == 3 {
vec!["eu".into(), "us".into(), "apac".into()]
} else {
(0..size).map(|i| format!("item{i:04}")).collect()
};
let definition = json!({
"operation_id": format!("op_{operation_name}"),
"operation_version": 1,
"uri_template": uri_template,
"parameters": [{
"name": "region",
"input_path": "input.region",
"schema": {
"type": "string",
"required": true,
"enum_values": options
}
}],
"cache": {"ttl_ms": 60_000, "vary": ["region"], "max_bytes": 65_536},
"failure": {"outcome": "error"}
});
let source = TemplateSource::parse(&definition)
.expect("the template fixture satisfies the Story 3.10 source contract");
let definition_json =
serde_json::to_string(&source).expect("the normalized definition serializes canonically");
let digest = format!("{:x}", Sha256::digest(definition_json.as_bytes()));
let draft = registry
.create_resource_template_draft(CreateResourceTemplateDraftRequest {
workspace_id: workspace_id(),
resource_id: resource_id.clone(),
uri: format!("crank://templates/{}", resource_id.as_str()),
name: format!("{}_template", resource_id.as_str()),
description: "Bound completion fixture template".to_owned(),
kind: "text".to_owned(),
mime_type: "application/json".to_owned(),
sensitivity: "internal".to_owned(),
definition_json: definition_json.clone(),
definition_digest: digest.clone(),
created_at: time::OffsetDateTime::now_utc(),
})
.await
.expect("the template Draft is created through the registry API");
registry
.publish_resource_template_version(PublishResourceTemplateVersionRequest {
workspace_id: workspace_id(),
resource_id: resource_id.clone(),
expected_updated_at: draft.snapshot.resource.updated_at,
expected_version: draft.snapshot.version.version,
expected_digest: digest,
published_at: time::OffsetDateTime::now_utc(),
})
.await
.expect("the template exact version publishes");
}
@@ -0,0 +1,409 @@
use super::*;
#[tokio::test]
async fn rmcp_http_catalog_follows_new_publication_without_reinitializing() {
let h = Harness::new().await;
let session = h.initialize().await;
let list = json!({"jsonrpc":"2.0","id":60,"method":"tools/list"});
let listed = frames(
h.request("2025-11-25", Some(&session), &list)
.send()
.await
.unwrap(),
)
.await;
assert_eq!(
listed.last().unwrap()["result"]["tools"][0]["name"],
"create_lead"
);
let workspace = crank_core::WorkspaceId::new("ws_default");
let agent = crank_core::AgentId::new("agent_dispatch-observer");
let now = time::OffsetDateTime::now_utc();
let version = crank_core::AgentVersion {
agent_id: agent.clone(),
version: 2,
status: crank_core::AgentStatus::Draft,
instructions: json!({}),
tool_selection_policy: crank_core::ToolSelectionPolicy::default(),
created_at: now,
};
let bindings = [crank_core::AgentOperationBinding {
operation_id: crank_core::OperationId::new("op_create_lead"),
operation_version: 1,
tool_name: "new_publication_tool".into(),
tool_title: "New Tool".into(),
tool_description_override: None,
enabled: true,
}];
h.state
.registry
.create_agent_draft_version(crank_registry::CreateAgentDraftVersionRequest {
workspace_id: &workspace,
agent_id: &agent,
version: &version,
bindings: &bindings,
updated_at: &now,
})
.await
.unwrap();
h.state
.registry
.publish_agent(crank_registry::PublishAgentRequest {
workspace_id: &workspace,
agent_id: &agent,
version: 2,
published_at: &now,
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
let listed = frames(
h.request("2025-11-25", Some(&session), &list)
.send()
.await
.unwrap(),
)
.await;
assert_eq!(
listed.last().unwrap()["result"]["tools"][0]["name"],
"new_publication_tool"
);
let mut message = call(61, "new-publication@example.test", false, false);
message["params"]["name"] = json!("new_publication_tool");
let result = frames(
h.request("2025-11-25", Some(&session), &message)
.send()
.await
.unwrap(),
)
.await;
assert_eq!(
result.last().unwrap()["result"]["structuredContent"]["id"],
"lead_for_new-publication@example.test"
);
assert_eq!(h.calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn rmcp_http_identical_request_ids_in_two_sessions_cancel_independently() {
let h = Harness::new().await;
let a = h.initialize().await;
let b = h.initialize().await;
let message = call(62, "slow-same-id@example.test", false, true);
let first = h
.request("2025-11-25", Some(&a), &message)
.send()
.await
.unwrap();
let second = h
.request("2025-11-25", Some(&b), &message)
.send()
.await
.unwrap();
wait_count(&h.calls, 2).await;
let cancel =
json!({"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":62}});
assert_eq!(
h.request("2025-11-25", Some(&a), &cancel)
.send()
.await
.unwrap()
.status(),
202
);
wait_count(&h.active, 1).await;
assert_eq!(
h.request("2025-11-25", Some(&b), &cancel)
.send()
.await
.unwrap()
.status(),
202
);
wait_count(&h.active, 0).await;
for response in [first, second] {
assert_eq!(
frames(response).await.last().unwrap()["error"]["message"],
"request cancelled"
);
}
assert_eq!(h.calls.load(Ordering::SeqCst), 2);
}
// Independent QA: dropped legacy response bodies must not bypass the stream ceiling.
#[tokio::test]
async fn rmcp_http_qa_legacy_disconnect_does_not_bypass_admission() {
let h = Harness::new().await;
let session = h.initialize().await;
let mut admitted = Vec::new();
for id in 100..117 {
let response = h
.request(
"2025-11-25",
Some(&session),
&call(id, "slow-qa-limit@example.test", false, true),
)
.send()
.await
.unwrap();
if response.status() == 429 {
break;
}
assert_eq!(response.status(), 200);
admitted.push(id);
wait_count(&h.calls, admitted.len()).await;
drop(response);
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
let active_before_cleanup = h.active.load(Ordering::SeqCst);
let free_slots = h.state.session_slots.available_permits();
let calls_before_cleanup = h.calls.load(Ordering::SeqCst);
for id in &admitted {
let cancel =
json!({"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":id}});
assert_eq!(
h.request("2025-11-25", Some(&session), &cancel)
.send()
.await
.unwrap()
.status(),
202
);
}
wait_count(&h.active, 0).await;
assert!(
admitted.len() <= 16 && calls_before_cleanup <= 16 && active_before_cleanup <= 16,
"legacy disconnect bypassed admission: admitted={}, calls={calls_before_cleanup}, active={active_before_cleanup}, free_slots={free_slots}",
admitted.len()
);
tokio::time::timeout(std::time::Duration::from_secs(2), async {
while h.state.session_slots.available_permits() != 16 {
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
})
.await
.expect("owned cancellation cleanup must release all admission permits");
assert_eq!(h.state.session_slots.available_permits(), 16);
// After cleanup, normal admission and REST execution must work again.
let calls_before = h.calls.load(Ordering::SeqCst);
let message = call(199, "recovery-after-cleanup@example.test", false, false);
let response = h
.request("2025-11-25", Some(&session), &message)
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
let result = frames(response).await;
assert_eq!(
result.last().unwrap()["result"]["structuredContent"]["id"],
"lead_for_recovery-after-cleanup@example.test"
);
assert_eq!(h.calls.load(Ordering::SeqCst), calls_before + 1);
}
#[tokio::test]
async fn rmcp_http_correlation_headers_preserve_each_concurrent_request() {
let h = Harness::new().await;
let first = call(50, "correlation-first@example.test", true, false);
let second = call(51, "correlation-second@example.test", true, false);
let (a, b) = tokio::join!(
h.request("2026-07-28", None, &first)
.header("x-request-id", "rmcp-oracle-first")
.send(),
h.request("2026-07-28", None, &second)
.header("x-request-id", "rmcp-oracle-second")
.send()
);
let a = a.unwrap();
let b = b.unwrap();
assert_eq!(a.headers()["x-request-id"], "rmcp-oracle-first");
assert_eq!(b.headers()["x-request-id"], "rmcp-oracle-second");
assert_eq!(
frames(a).await[0]["result"]["structuredContent"]["id"],
"lead_for_correlation-first@example.test"
);
assert_eq!(
frames(b).await[0]["result"]["structuredContent"]["id"],
"lead_for_correlation-second@example.test"
);
}
#[tokio::test]
async fn rmcp_http_stream_limit_rejects_without_dispatch_then_recovers() {
let h = Harness::new().await;
let permits = h
.state
.session_slots
.clone()
.acquire_many_owned(16)
.await
.unwrap();
let message = call(40, "limit@example.test", true, false);
assert_eq!(
h.request("2026-07-28", None, &message)
.send()
.await
.unwrap()
.status(),
429
);
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
drop(permits);
let response = h
.request("2026-07-28", None, &message)
.send()
.await
.unwrap();
assert_eq!(
frames(response).await[0]["result"]["structuredContent"]["id"],
"lead_for_limit@example.test"
);
assert_eq!(h.calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn rmcp_http_rate_limit_is_existing_admission_not_sdk_default() {
let h = Harness::with_rate(1).await;
let list = json!({"jsonrpc":"2.0","id":41,"method":"tools/list","params":{"_meta":meta()}});
assert_eq!(
h.request("2026-07-28", None, &list)
.send()
.await
.unwrap()
.status(),
200
);
assert_eq!(
h.request(
"2026-07-28",
None,
&call(42, "rate@example.test", true, false)
)
.send()
.await
.unwrap()
.status(),
429
);
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn rmcp_http_current_durable_store_survives_recreation_but_is_not_sdk_store() {
use crate::session::{PostgresTransportSessionStore, TransportSessionStore};
let h = Harness::new().await;
let store = PostgresTransportSessionStore::from_pool(h.state.registry.pool().clone())
.await
.unwrap();
let now = time::OffsetDateTime::now_utc();
let id = store
.create(
"2025-11-25",
"default",
"dispatch-observer",
false,
now,
Some(now + time::Duration::hours(24)),
)
.await
.unwrap();
assert!(store.mark_initialized(&id, now).await.unwrap());
drop(store);
let restarted = PostgresTransportSessionStore::from_pool(h.state.registry.pool().clone())
.await
.unwrap();
let persisted = restarted.get(&id).await.unwrap().unwrap();
assert!(persisted.initialized);
assert_eq!(persisted.workspace_slug, "default");
assert_eq!(persisted.protocol_version, "2025-11-25");
assert_eq!(
h.request(
"2025-11-25",
Some(&id),
&call(43, "durable@example.test", false, false)
)
.send()
.await
.unwrap()
.status(),
404
);
assert!(
restarted.get(&id).await.unwrap().is_some(),
"SDK experiment never mutates existing storage"
);
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn rmcp_http_json_modern_and_legacy_sse_only_gap_are_observed() {
let h = Harness::new().await;
let session = h.initialize().await;
let response = h
.request(
"2026-07-28",
None,
&call(44, "json@example.test", true, false),
)
.send()
.await
.unwrap();
assert!(
response.headers()["content-type"]
.to_str()
.unwrap()
.starts_with("application/json")
);
assert_eq!(
frames(response).await[0]["result"]["structuredContent"]["id"],
"lead_for_json@example.test"
);
// Build a JSON-only request (not two contradictory Accept carriers).
let response = reqwest::Client::new()
.post(format!("{}/v1/default/dispatch-observer", h.server.1))
.bearer_auth(&h.key)
.header("Accept", "application/json")
.header("MCP-Protocol-Version", "2025-11-25")
.header("MCP-Session-Id", &session)
.json(&call(45, "legacy-json@example.test", false, false))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
406,
"bundled SDK legacy mode requires SSE Accept; not transport parity"
);
assert_eq!(h.calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn rmcp_http_concurrent_keys_keep_outputs_and_cancel_ownership_separate() {
let h = Harness::new().await;
let session = h.initialize().await;
sqlx::query("insert into platform_api_keys (id, workspace_id, agent_id, key_kind, name, prefix, secret_hash, scopes_json, status, created_at) select 'pk_second', workspace_id, agent_id, key_kind, 'second', prefix, $1, scopes_json, status, created_at from platform_api_keys where id = 'pk_dispatch_observer'")
.bind(crate::access::hash_access_secret("test-second-valid-key")).execute(h.state.registry.pool()).await.unwrap();
let first = call(46, "first@example.test", false, false);
let second = call(47, "second@example.test", false, false);
let (a, b) = tokio::join!(
h.request("2025-11-25", Some(&session), &first).send(),
h.request_as(
"2025-11-25",
Some(&session),
&second,
"test-second-valid-key"
)
.send()
);
assert_eq!(
frames(a.unwrap()).await.last().unwrap()["result"]["structuredContent"]["id"],
"lead_for_first@example.test"
);
assert_eq!(
frames(b.unwrap()).await.last().unwrap()["result"]["structuredContent"]["id"],
"lead_for_second@example.test"
);
assert_eq!(h.calls.load(Ordering::SeqCst), 2);
}
@@ -0,0 +1,563 @@
//! Wire oracle for the isolated SDK adapter, with real registry and REST execution.
#[path = "rmcp_http_limits.rs"]
mod limits;
use super::{AgentRoutePath, dispatch_observer::support, rmcp_http, rmcp_tools_tests::test_state};
use axum::{Json, Router, extract::State, routing::post};
use serde_json::{Value, json};
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
struct Server(tokio::task::JoinHandle<()>, String);
impl Drop for Server {
fn drop(&mut self) {
self.0.abort();
}
}
async fn serve(router: Router) -> Server {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
Server(
tokio::spawn(async move {
axum::serve(listener, router).await.unwrap();
}),
url,
)
}
async fn upstream(State(count): State<Arc<AtomicUsize>>, Json(body): Json<Value>) -> Json<Value> {
count.fetch_add(1, Ordering::SeqCst);
if body["email"].as_str().unwrap().starts_with("slow") {
tokio::time::sleep(std::time::Duration::from_secs(8)).await;
}
Json(json!({"id": format!("lead_for_{}", body["email"].as_str().unwrap())}))
}
struct Harness {
server: Server,
_upstream: Server,
key: String,
calls: Arc<AtomicUsize>,
state: Arc<super::AppState>,
active: Arc<AtomicUsize>,
}
impl Harness {
async fn new() -> Self {
Self::with_rate(10_000).await
}
async fn with_rate(burst: u32) -> Self {
let mut state = test_state().await;
Arc::get_mut(&mut state).unwrap().api_rate_limiter = crank_runtime::RequestRateLimiter::new(
crank_runtime::RequestRateLimitConfig::new(burst, burst).unwrap(),
);
let calls = Arc::new(AtomicUsize::new(0));
let upstream = serve(
Router::new()
.route("/crm/leads", post(upstream))
.with_state(calls.clone()),
)
.await;
let mut operation = support::test_operation(&upstream.1);
operation.execution_config.timeout_ms = 10_000;
state
.registry
.create_operation(
&crank_core::WorkspaceId::new("ws_default"),
&operation,
Some("alice"),
)
.await
.unwrap();
state
.registry
.publish_operation(crank_registry::PublishRequest {
workspace_id: &crank_core::WorkspaceId::new("ws_default"),
operation_id: &operation.id,
version: 1,
published_at: &operation.created_at,
published_by: Some("alice"),
})
.await
.unwrap();
support::publish_agent(&state.registry, &operation).await;
let key = support::create_api_key(&state.registry).await;
let active = Arc::new(AtomicUsize::new(0));
let server = serve(rmcp_http::router(
state.clone(),
AgentRoutePath {
workspace_slug: "default".into(),
agent_slug: "dispatch-observer".into(),
},
active.clone(),
))
.await;
Self {
server,
_upstream: upstream,
key,
calls,
state,
active,
}
}
fn request(
&self,
revision: &str,
session: Option<&str>,
message: &Value,
) -> reqwest::RequestBuilder {
self.request_as(revision, session, message, &self.key)
}
fn request_as(
&self,
revision: &str,
session: Option<&str>,
message: &Value,
key: &str,
) -> reqwest::RequestBuilder {
let mut request = reqwest::Client::new()
.post(format!("{}/v1/default/dispatch-observer", self.server.1))
.bearer_auth(key)
.header("Accept", "application/json, text/event-stream")
.header("MCP-Protocol-Version", revision);
if let Some(session) = session {
request = request.header("MCP-Session-Id", session);
}
if revision == "2026-07-28" {
request = request.header("Mcp-Method", message["method"].as_str().unwrap());
if let Some(name) = message.pointer("/params/name").and_then(Value::as_str) {
request = request.header("Mcp-Name", name);
}
}
request
.json(message)
.timeout(std::time::Duration::from_secs(10))
}
async fn initialize(&self) -> String {
let message = json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"oracle","version":"1"}}});
let response = self
.request("2025-11-25", None, &message)
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
let session = response.headers()["MCP-Session-Id"]
.to_str()
.unwrap()
.to_owned();
let body = frames(response).await;
assert_eq!(
body.last().unwrap()["result"]["protocolVersion"],
"2025-11-25"
);
let response = self
.request(
"2025-11-25",
Some(&session),
&json!({"jsonrpc":"2.0","method":"notifications/initialized"}),
)
.send()
.await
.unwrap();
assert_eq!(response.status(), 202);
session
}
}
fn meta() -> Value {
json!({"io.modelcontextprotocol/protocolVersion":"2026-07-28","io.modelcontextprotocol/clientCapabilities":{},"io.modelcontextprotocol/clientInfo":{"name":"oracle","version":"1"}})
}
async fn frames(response: reqwest::Response) -> Vec<Value> {
let status = response.status();
let text = response.text().await.unwrap();
if let Ok(json) = serde_json::from_str(&text) {
return vec![json];
}
let values: Vec<Value> = text
.lines()
.filter_map(|line| line.strip_prefix("data:"))
.filter_map(|data| serde_json::from_str(data.trim()).ok())
.collect();
assert!(!values.is_empty(), "{status}: {text}");
values
}
#[tokio::test]
async fn rmcp_http_positive_tools_both_revisions() {
let h = Harness::new().await;
let session = h.initialize().await;
for (revision, session) in [("2025-11-25", Some(session.as_str())), ("2026-07-28", None)] {
if session.is_none() {
let discover = json!({"jsonrpc":"2.0","id":2,"method":"server/discover","params":{"_meta":meta()}});
let response = h.request(revision, None, &discover).send().await.unwrap();
assert_eq!(response.status(), 200);
assert!(!frames(response).await[0]["result"].is_null());
}
let list = json!({"jsonrpc":"2.0","id":3,"method":"tools/list","params":{"_meta": if session.is_none() { meta() } else { json!({}) }}});
let response = h.request(revision, session, &list).send().await.unwrap();
assert_eq!(response.status(), 200);
assert!(response.headers().contains_key("x-request-id"));
let listed = frames(response).await;
assert_eq!(
listed.last().unwrap()["result"]["tools"][0]["name"],
"create_lead"
);
let call = json!({"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"create_lead","arguments":{"email":"rmcp@example.test"},"_meta":if session.is_none() { meta() } else { json!({}) }}});
let response = h.request(revision, session, &call).send().await.unwrap();
assert_eq!(response.status(), 200);
if session.is_none() {
assert!(!response.headers().contains_key("MCP-Session-Id"));
}
let result = frames(response).await;
assert_eq!(
result.last().unwrap()["result"]["structuredContent"]["id"],
"lead_for_rmcp@example.test",
"{result:?}"
);
}
assert_eq!(h.calls.load(Ordering::SeqCst), 2);
}
fn call(id: i64, email: &str, modern: bool, progress: bool) -> Value {
let mut metadata = if modern { meta() } else { json!({}) };
if progress {
metadata["progressToken"] = json!("wire-progress");
}
json!({"jsonrpc":"2.0","id":id,"method":"tools/call","params":{"name":"create_lead","arguments":{"email":email},"_meta":metadata}})
}
#[tokio::test]
async fn rmcp_http_credentials_revalidated_after_initialize_and_no_outbound() {
let h = Harness::new().await;
let session = h.initialize().await;
let message = call(10, "denied@example.test", false, false);
assert_eq!(
h.request_as("2025-11-25", Some(&session), &message, "invalid-test-key")
.send()
.await
.unwrap()
.status(),
401
);
// A scope change and expiry are authoritative registry changes, not session credentials.
sqlx::query("update platform_api_keys set scopes_json = '[\"read\"]'::jsonb where id = 'pk_dispatch_observer'").execute(h.state.registry.pool()).await.unwrap();
assert_eq!(
h.request("2025-11-25", Some(&session), &message)
.send()
.await
.unwrap()
.status(),
403
);
sqlx::query("update platform_api_keys set scopes_json = '[\"read\",\"write\"]'::jsonb, expires_at = now() - interval '1 second' where id = 'pk_dispatch_observer'").execute(h.state.registry.pool()).await.unwrap();
assert_eq!(
h.request("2025-11-25", Some(&session), &message)
.send()
.await
.unwrap()
.status(),
401
);
sqlx::query("update platform_api_keys set expires_at = null where id = 'pk_dispatch_observer'")
.execute(h.state.registry.pool())
.await
.unwrap();
h.state
.registry
.revoke_platform_api_key(
&crank_core::WorkspaceId::new("ws_default"),
&crank_core::PlatformApiKeyId::new("pk_dispatch_observer"),
&time::OffsetDateTime::now_utc(),
)
.await
.unwrap();
for method in [reqwest::Method::GET, reqwest::Method::DELETE] {
assert_eq!(
reqwest::Client::new()
.request(
method,
format!("{}/v1/default/dispatch-observer", h.server.1)
)
.bearer_auth(&h.key)
.header("MCP-Session-Id", &session)
.header("MCP-Protocol-Version", "2025-11-25")
.header("Accept", "text/event-stream")
.send()
.await
.unwrap()
.status(),
401
);
}
assert_eq!(
h.request("2025-11-25", Some(&session), &message)
.send()
.await
.unwrap()
.status(),
401
);
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn rmcp_http_admission_rejects_foreign_route_revision_headers_and_body_overflow() {
let h = Harness::new().await;
let message = call(11, "denied@example.test", true, false);
let response = h
.request("2026-07-28", Some("foreign"), &message)
.send()
.await
.unwrap();
assert_eq!(response.status(), 400);
let response = h
.request("2026-07-28", None, &message)
.header("Mcp-Name", "foreign")
.send()
.await
.unwrap();
assert_eq!(response.status(), 400);
let response = h
.request("2026-07-28", None, &message)
.header("Origin", "https://evil.example")
.send()
.await
.unwrap();
assert_eq!(response.status(), 403);
assert_eq!(
h.request("2030-01-01", None, &message)
.send()
.await
.unwrap()
.status(),
400
);
let response = reqwest::Client::new()
.post(format!("{}/v1/default/foreign-agent", h.server.1))
.bearer_auth(&h.key)
.json(&message)
.send()
.await
.unwrap();
assert_eq!(response.status(), 404);
let response = h
.request("2026-07-28", None, &message)
.body(vec![b' '; 2 * 1024 * 1024 + 1])
.send()
.await
.unwrap();
assert_eq!(response.status(), 413);
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn rmcp_http_progress_is_real_and_tool_error_is_not_protocol_error() {
let h = Harness::new().await;
let session = h.initialize().await;
for (revision, session) in [("2025-11-25", Some(session.as_str())), ("2026-07-28", None)] {
let response = h
.request(
revision,
session,
&call(20, "progress@example.test", session.is_none(), true),
)
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
assert!(
response.headers()["content-type"]
.to_str()
.unwrap()
.starts_with("text/event-stream")
);
let messages = frames(response).await;
let progress: Vec<_> = messages
.iter()
.filter(|m| m["method"] == "notifications/progress")
.collect();
assert_eq!(progress.len(), 3, "{messages:?}");
for (index, phase) in [
"tool_resolved",
"authorization_resolved",
"execution_dispatched",
]
.iter()
.enumerate()
{
assert_eq!(progress[index]["params"]["message"], *phase);
assert_eq!(
progress[index]["params"]["progress"].as_f64(),
Some((index + 1) as f64)
);
assert_eq!(progress[index]["params"]["progressToken"], "wire-progress");
}
let mut invalid = call(21, "unused", session.is_none(), false);
invalid["params"]["arguments"] = json!({"email":123});
let response = h.request(revision, session, &invalid).send().await.unwrap();
let messages = frames(response).await;
let result = messages.last().unwrap();
assert_eq!(result["result"]["isError"], true, "{messages:?}");
assert!(result.get("error").is_none());
}
assert_eq!(h.calls.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn rmcp_http_reconnect_live_manager_and_restart_loses_sdk_session() {
let h = Harness::new().await;
let session = h.initialize().await;
for _ in 0..2 {
let response = reqwest::Client::new()
.get(format!("{}/v1/default/dispatch-observer", h.server.1))
.bearer_auth(&h.key)
.header("MCP-Session-Id", &session)
.header("MCP-Protocol-Version", "2025-11-25")
.header("Accept", "text/event-stream")
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
drop(response);
}
let response = h
.request(
"2025-11-25",
Some(&session),
&call(22, "reconnect@example.test", false, false),
)
.send()
.await
.unwrap();
assert_eq!(
frames(response).await.last().unwrap()["result"]["structuredContent"]["id"],
"lead_for_reconnect@example.test"
);
let fresh = serve(rmcp_http::router(
h.state.clone(),
AgentRoutePath {
workspace_slug: "default".into(),
agent_slug: "dispatch-observer".into(),
},
h.active.clone(),
))
.await;
let response = reqwest::Client::new()
.post(format!("{}/v1/default/dispatch-observer", fresh.1))
.bearer_auth(&h.key)
.header("MCP-Session-Id", &session)
.header("MCP-Protocol-Version", "2025-11-25")
.header("Accept", "application/json, text/event-stream")
.json(&call(23, "restart@example.test", false, false))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
404,
"new SDK manager cannot restore from Crank session storage"
);
assert_eq!(h.calls.load(Ordering::SeqCst), 1);
}
async fn wait_count(counter: &AtomicUsize, expected: usize) {
tokio::time::timeout(std::time::Duration::from_secs(3), async {
while counter.load(Ordering::SeqCst) != expected {
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
})
.await
.expect("bounded wait for the real execution/cleanup");
}
#[tokio::test]
async fn rmcp_http_legacy_cancel_is_owned_by_session_and_current_key() {
let h = Harness::new().await;
let session = h.initialize().await;
let foreign_session = h.initialize().await;
// A second valid credential can use this Agent/session but does not own the first call.
sqlx::query("insert into platform_api_keys (id, workspace_id, agent_id, key_kind, name, prefix, secret_hash, scopes_json, status, created_at) select 'pk_second', workspace_id, agent_id, key_kind, 'second', prefix, $1, scopes_json, status, created_at from platform_api_keys where id = 'pk_dispatch_observer'")
.bind(crate::access::hash_access_secret("test-second-valid-key")).execute(h.state.registry.pool()).await.unwrap();
let response = h
.request(
"2025-11-25",
Some(&session),
&call(30, "slow-cancel@example.test", false, true),
)
.send()
.await
.unwrap();
wait_count(&h.calls, 1).await;
let cancel =
json!({"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":30}});
assert_eq!(
h.request_as(
"2025-11-25",
Some(&session),
&cancel,
"test-second-valid-key"
)
.send()
.await
.unwrap()
.status(),
202
);
assert_eq!(
h.request("2025-11-25", Some(&foreign_session), &cancel)
.send()
.await
.unwrap()
.status(),
202
);
tokio::time::sleep(std::time::Duration::from_millis(75)).await;
assert_eq!(
h.active.load(Ordering::SeqCst),
1,
"foreign cancel must not stop call"
);
assert_eq!(
h.request("2025-11-25", Some(&session), &cancel)
.send()
.await
.unwrap()
.status(),
202
);
wait_count(&h.active, 0).await;
// Gap: SDK emits an error frame where the existing transport suppresses all terminal frames.
let messages = frames(response).await;
assert_eq!(
messages.last().unwrap()["error"]["message"],
"request cancelled"
);
assert!(!messages.iter().any(|m| m.get("result").is_some()));
}
#[tokio::test]
async fn rmcp_http_modern_disconnect_drops_execution_and_releases_stream_slot() {
let h = Harness::new().await;
let response = h
.request(
"2026-07-28",
None,
&call(31, "slow-disconnect@example.test", true, true),
)
.send()
.await
.unwrap();
wait_count(&h.calls, 1).await;
assert_eq!(h.active.load(Ordering::SeqCst), 1);
drop(response);
wait_count(&h.active, 0).await;
tokio::time::timeout(std::time::Duration::from_secs(2), async {
while h.state.session_slots.available_permits() != 16 {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
assert_eq!(
h.calls.load(Ordering::SeqCst),
1,
"disconnect never retries a possible external effect"
);
}
@@ -0,0 +1,258 @@
//! #107 logging: existing HTTP wire control before the independent SDK marker assertions.
use super::{CounterSet, observe, support::*};
use axum::{body::Body, response::Response};
use futures_util::{StreamExt, stream};
use serde_json::{Value, json};
use std::{sync::Arc, time::Duration};
#[path = "rmcp_logging_test_support.rs"]
mod fixtures;
use fixtures::Endpoint;
const LEVEL: &str = "io.modelcontextprotocol/logLevel";
fn counts(c: &CounterSet) -> [usize; 3] {
[
c.sdk_logging_dispatch(),
c.sdk_logging_level(),
c.sdk_logging_notification(),
]
}
fn observe_body(response: Response, counters: Arc<CounterSet>) -> Response {
let (parts, body) = response.into_parts();
let body = stream::unfold(
(body.into_data_stream(), counters),
|(mut body, c)| async move {
observe(c.clone(), body.next())
.await
.map(|chunk| (chunk, (body, c)))
},
);
Response::from_parts(parts, Body::from_stream(body))
}
fn notification(level: &str, event: &str, outcome: &str) -> Value {
json!({"jsonrpc":"2.0","method":"notifications/message","params":{
"level":level,"logger":"crank-mcp-server","data":{"event":event,"outcome":outcome}}})
}
async fn reply(response: reqwest::Response) -> (Value, Vec<Value>, String) {
let status = response.status();
assert!(
status == 200 || status == 404,
"unexpected HTTP status {status}"
);
let sse = response.headers()["content-type"]
.to_str()
.unwrap()
.starts_with("text/event-stream");
let raw = tokio::time::timeout(Duration::from_secs(5), response.text())
.await
.unwrap()
.unwrap();
let mut frames: Vec<Value> = if sse {
raw.lines()
.filter_map(|line| {
line.strip_prefix("data:")
.map(|s| serde_json::from_str(s.trim()).unwrap())
})
.collect()
} else {
vec![serde_json::from_str(&raw).unwrap()]
};
let terminal = frames.pop().expect("one terminal response");
assert_eq!(terminal["jsonrpc"], "2.0");
assert_eq!(terminal["id"], 51);
assert_eq!(
status.as_u16(),
if terminal["error"]["code"] == -32601 {
404
} else {
200
}
);
if !frames.is_empty() {
assert!(sse, "notifications require SSE");
}
assert!(frames.iter().all(|f| f.get("id").is_none()));
(terminal, frames, raw)
}
fn opted(level: Value) -> Value {
let mut meta = modern_meta();
meta[LEVEL] = level;
json!({"_meta":meta})
}
async fn control(revision: &str) -> [usize; 3] {
let h = Endpoint::started().await;
let session = h.session(revision).await;
let session = session.as_deref();
// Negative opt-in controls precede any accepted level; a silent response is not enough alone.
let (body, events, _) = reply(h.post(revision, session, "tools/list", json!({})).await).await;
assert!(body["result"]["tools"].is_array());
assert!(events.is_empty());
assert_eq!(counts(&h.counters), [0; 3]);
if revision == MODERN_REVISION {
for value in [
json!("verbose"),
json!(7),
json!(null),
json!("CANARY".repeat(1000)),
] {
let (body, events, _) =
reply(h.post(revision, session, "tools/list", opted(value)).await).await;
assert!(body["result"]["tools"].is_array());
assert!(events.is_empty());
assert_eq!(counts(&h.counters), [0; 3]);
}
let (body, events, _) = reply(
h.post(
revision,
session,
"logging/setLevel",
json!({"level":"notice"}),
)
.await,
)
.await;
assert_eq!(body["error"]["code"], -32601);
assert!(events.is_empty());
assert_eq!(counts(&h.counters), [0; 3]);
} else {
for params in [
json!({}),
json!({"level":7}),
json!({"level":"CANARY".repeat(1000)}),
] {
let (body, events, raw) =
reply(h.post(revision, session, "logging/setLevel", params).await).await;
assert_eq!(body["error"]["code"], -32602);
assert_eq!(body["error"]["message"], "invalid logging level");
assert!(body.get("result").is_none());
assert!(!raw.contains("CANARY"));
assert!(raw.len() < 4096);
assert!(events.is_empty());
assert_eq!(h.counters.sdk_logging_level(), 0);
assert_eq!(h.counters.sdk_logging_notification(), 0);
}
}
let before = counts(&h.counters);
let (method, params, expected) = if revision == LEGACY_REVISION {
(
"logging/setLevel",
json!({"level":"notice"}),
notification("notice", "mcp.logging.level_applied", "applied"),
)
} else {
(
"tools/list",
opted(json!("info")),
notification("info", "mcp.request.served", "ok"),
)
};
let (body, events, raw) = reply(h.post(revision, session, method, params).await).await;
assert!(body.get("error").is_none(), "{body}");
if revision == LEGACY_REVISION {
assert_eq!(body["result"], json!({}));
} else {
assert!(body["result"]["tools"].is_array());
}
assert_eq!(events, vec![expected]);
assert!(raw.len() < 4096);
let after = counts(&h.counters);
let delta = std::array::from_fn(|i| after[i] - before[i]);
// Legacy severity persists, modern opt-in does not leak into the next request.
let (body, events, _) = reply(h.post(revision, session, "tools/list", json!({})).await).await;
assert!(body["result"]["tools"].is_array());
assert!(events.is_empty());
h.registry
.revoke_platform_api_key(
&workspace_id(),
&crank_core::PlatformApiKeyId::new("pk_dispatch_observer"),
&time::OffsetDateTime::now_utc(),
)
.await
.unwrap();
let before = counts(&h.counters);
let era = h.counters.snapshot();
let response = h
.post(
revision,
session,
method,
if revision == LEGACY_REVISION {
json!({"level":"notice"})
} else {
opted(json!("info"))
},
)
.await;
assert_eq!(response.status(), 401);
let raw = response.text().await.unwrap();
assert!(!raw.contains("notifications/message"));
assert_eq!(counts(&h.counters), before);
let after = h.counters.snapshot();
assert_eq!(after.era_modern, era.era_modern);
assert_eq!(after.era_legacy, era.era_legacy);
eprintln!(
"existing GREEN logging HTTP {revision}: safe SSE, result, opt-in/invalid/era/auth controls passed; SDK positive delta={delta:?}"
);
delta
}
#[tokio::test]
async fn rmcp_logging_wire_legacy() {
control(LEGACY_REVISION).await;
}
#[tokio::test]
async fn rmcp_logging_wire_modern() {
control(MODERN_REVISION).await;
}
#[tokio::test]
async fn rmcp_logging_sdk_legacy() {
assert_eq!(
control(LEGACY_REVISION).await,
[1, 1, 1],
"missing SDK legacy request/level/notification after successful HTTP wire assertions"
);
}
#[tokio::test]
async fn rmcp_logging_sdk_modern() {
assert_eq!(
control(MODERN_REVISION).await,
[0, 1, 1],
"missing SDK modern level/notification after successful HTTP wire assertions"
);
}
#[tokio::test]
async fn rmcp_logging_observer_scope_control() {
use super::{
sdk_logging_dispatch_enter, sdk_logging_level_parsed, sdk_logging_notification_serialized,
};
let a = Arc::new(CounterSet::default());
let b = Arc::new(CounterSet::default());
sdk_logging_dispatch_enter();
sdk_logging_level_parsed();
sdk_logging_notification_serialized();
assert_eq!(counts(&a), [0; 3]);
tokio::join!(
observe(a.clone(), async {
sdk_logging_dispatch_enter();
tokio::task::yield_now().await;
sdk_logging_level_parsed();
}),
observe(b.clone(), async {
sdk_logging_notification_serialized();
})
);
assert_eq!(counts(&a), [1, 1, 0]);
assert_eq!(counts(&b), [0, 0, 1]);
let body = Body::from_stream(stream::once(async {
sdk_logging_notification_serialized();
Ok::<_, std::convert::Infallible>("data: terminal\n\n")
}));
let response = observe_body(Response::new(body), a.clone());
assert_eq!(counts(&a), [1, 1, 0]);
assert_eq!(
&axum::body::to_bytes(response.into_body(), 100)
.await
.unwrap()[..],
b"data: terminal\n\n"
);
assert_eq!(counts(&a), [1, 1, 1]);
assert_eq!(counts(&b), [0, 0, 1]);
}
@@ -0,0 +1,143 @@
//! Minimal logging HTTP fixture; shared registry/auth/lifecycle fixtures remain authoritative.
use super::*;
use crank_runtime::{
InMemoryCoordinationStateStore, RequestRateLimitConfig, RequestRateLimiter, SecretCrypto,
};
pub(super) struct Endpoint {
pub registry: crank_registry::PostgresRegistry,
pub counters: Arc<CounterSet>,
key: String,
url: String,
server: tokio::task::JoinHandle<()>,
}
impl Drop for Endpoint {
fn drop(&mut self) {
self.server.abort();
}
}
impl Endpoint {
pub async fn started() -> Self {
crank_test_support::postgres_schema_url_for_task("logging_owner", "t_c28c3800").await;
let registry = test_registry().await;
let operation = test_operation("http://127.0.0.1:9");
registry
.create_operation(&workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(crank_registry::PublishRequest {
workspace_id: &workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &test_time(),
published_by: Some("alice"),
})
.await
.unwrap();
publish_agent(&registry, &operation).await;
let key = create_api_key(&registry).await;
let counters = Arc::new(CounterSet::default());
let app = crate::build_app(
registry.clone(),
Duration::ZERO,
Some("https://crank.example.com".into()),
SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(),
crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build(),
RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()),
Arc::new(InMemoryCoordinationStateStore::default()),
Arc::new(crate::session::InMemorySessionStore::default()),
Arc::new(crate::auth::CommunityMachineCredentialVerifier),
);
let scope = counters.clone();
let app = app.layer(axum::middleware::from_fn(
move |request: axum::extract::Request, next: axum::middleware::Next| {
let counters = scope.clone();
async move {
let response = observe(counters.clone(), next.run(request)).await;
observe_body(response, counters)
}
},
));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!(
"http://{}/v1/{WORKSPACE_SLUG}/{AGENT_SLUG}",
listener.local_addr().unwrap()
);
let server = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
Self {
registry,
counters,
key,
url,
server,
}
}
pub async fn post(
&self,
revision: &str,
session: Option<&str>,
method: &str,
mut params: Value,
) -> reqwest::Response {
let mut request = reqwest::Client::new()
.post(&self.url)
.bearer_auth(&self.key)
.header("Accept", "text/event-stream, application/json")
.header("MCP-Protocol-Version", revision);
if let Some(session) = session {
request = request.header("MCP-Session-Id", session);
}
if revision == MODERN_REVISION {
request = request.header("Mcp-Method", method);
if params.get("_meta").is_none() {
params["_meta"] = modern_meta();
}
}
request
.json(&json!({"jsonrpc":"2.0", "id":51, "method":method, "params":params}))
.send()
.await
.unwrap()
}
pub async fn session(&self, revision: &str) -> Option<String> {
if revision == MODERN_REVISION {
return None;
}
let response = self
.post(
revision,
None,
"initialize",
json!({"protocolVersion":revision,"capabilities":{}}),
)
.await;
assert_eq!(response.status(), 200);
let session = response.headers()["MCP-Session-Id"]
.to_str()
.unwrap()
.to_owned();
assert_eq!(
reply(response).await.0["result"]["protocolVersion"],
revision
);
let response = reqwest::Client::new()
.post(&self.url)
.bearer_auth(&self.key)
.header("Accept", "application/json, text/event-stream")
.header("MCP-Protocol-Version", revision)
.header("MCP-Session-Id", &session)
.json(&initialized_notification())
.send()
.await
.unwrap();
assert_eq!(response.status(), 202);
response.bytes().await.unwrap();
Some(session)
}
}
@@ -0,0 +1,436 @@
//! #107 Prompts: existing wire GREEN and an independent SDK typed-path RED oracle.
use super::{CounterSet, observe, support::*};
use axum::{body::Body, response::Response};
use futures_util::{StreamExt, stream};
use serde_json::{Value, json};
use std::{sync::Arc, time::Duration};
#[path = "rmcp_prompts_test_support.rs"]
mod fixtures;
use fixtures::Endpoint;
fn observe_body(response: Response, counters: Arc<CounterSet>) -> Response {
let (parts, body) = response.into_parts();
let body = stream::unfold(
(body.into_data_stream(), counters),
|(mut body, counters)| async move {
observe(counters.clone(), body.next())
.await
.map(|chunk| (chunk, (body, counters)))
},
);
Response::from_parts(parts, Body::from_stream(body))
}
fn counts(c: &CounterSet) -> [usize; 3] {
[
c.sdk_prompts_dispatch(),
c.sdk_prompts_list_result(),
c.sdk_prompts_get_result(),
]
}
fn request(revision: &str, method: &str, mut params: Value) -> Value {
if revision == MODERN_REVISION {
params["_meta"] = modern_meta();
}
json!({"jsonrpc":"2.0","id":9007199254740993_u64,"method":method,"params":params})
}
async fn wire(response: reqwest::Response, id: &Value, sse: bool) -> Value {
assert_eq!(response.status(), 200);
assert_eq!(response.headers()["x-request-id"], "sdk-prompts-wire");
assert_eq!(
response.headers()["x-trace-id"],
"11111111111111111111111111111111"
);
let content_type = response.headers()["content-type"]
.to_str()
.unwrap()
.to_owned();
let text = tokio::time::timeout(Duration::from_secs(5), response.text())
.await
.unwrap()
.unwrap();
let body: Value = if sse {
assert!(
content_type.starts_with("text/event-stream"),
"{content_type}: {text}"
);
let frames: Vec<Value> = text
.lines()
.filter_map(|line| {
line.strip_prefix("data:")
.map(|s| serde_json::from_str(s.trim()).unwrap())
})
.collect();
let terminal: Vec<_> = frames.iter().filter(|f| f.get("id") == Some(id)).collect();
assert_eq!(terminal.len(), 1, "{frames:?}");
assert_eq!(frames.last(), Some(terminal[0]), "terminal must be last");
terminal[0].clone()
} else {
assert!(
content_type.starts_with("application/json"),
"{content_type}: {text}"
);
serde_json::from_str(&text).unwrap()
};
assert_eq!(body["jsonrpc"], "2.0");
assert_eq!(&body["id"], id);
body
}
fn era(result: &Value, revision: &str, method: &str) {
if revision == MODERN_REVISION {
assert_eq!(result["resultType"], "complete", "{result}");
if method == "prompts/list" {
assert_eq!(result["ttlMs"], 0);
assert_eq!(result["cacheScope"], "private");
} else {
assert!(result.get("ttlMs").is_none(), "{result}");
assert!(result.get("cacheScope").is_none(), "{result}");
}
} else {
for key in ["resultType", "ttlMs", "cacheScope"] {
assert!(result.get(key).is_none(), "{result}");
}
}
assert!(result.get("_meta").is_none(), "{result}");
}
async fn success(
h: &Endpoint,
revision: &str,
session: Option<&str>,
method: &str,
params: Value,
sse: bool,
) -> (Value, [usize; 3]) {
let req = request(revision, method, params);
let before = counts(&h.counters);
let body = wire(h.post(revision, session, &req, sse).await, &req["id"], sse).await;
assert!(body.get("error").is_none(), "{body}");
era(&body["result"], revision, method);
let after = counts(&h.counters);
(
body["result"].clone(),
std::array::from_fn(|i| after[i] - before[i]),
)
}
async fn control(revision: &str) -> Vec<([usize; 3], [usize; 3])> {
let h = Endpoint::started().await;
let session = h.session(revision).await;
let session = session.as_deref();
let mut deltas = Vec::new();
let mut first_cursor = None;
// Both revisions' JSON contract, plus modern terminal SSE through the same normal app.
let modes: &[bool] = if revision == MODERN_REVISION {
&[false, true]
} else {
&[false]
};
for &sse in modes {
let mut cursor = None;
for (page, name) in ["greeting", "archived-prompt"].iter().enumerate() {
let mut params = json!({"limit":1});
if let Some(ref cursor) = cursor {
params["cursor"] = json!(cursor);
}
let (result, delta) = success(&h, revision, session, "prompts/list", params, sse).await;
let prompts = result["prompts"].as_array().unwrap();
assert_eq!(prompts.len(), 1, "{result}");
assert_eq!(
prompts[0],
json!({
"name":name, "description":format!("Published {name}"),
"arguments":[
{"name":"name","schema":{"type":"string"},"required":true},
{"name":"count","schema":{"type":"integer"},"required":false,"default":2},
{"name":"enabled","schema":{"type":"boolean"},"required":false,"default":true},
{"name":"tone","schema":{"type":"string","enum":["warm","dry"]},"required":false,"default":"warm"}
]
})
);
cursor = result
.get("nextCursor")
.map(|v| v.as_str().unwrap().to_owned());
assert_eq!(cursor.is_some(), page == 0, "{result}");
if page == 0 {
assert_ne!(cursor.as_deref(), Some("1"));
first_cursor = cursor.clone();
}
deltas.push((delta, [1, 1, 0]));
}
for (name, args, text) in [
(
"greeting",
json!({"name":"Ada"}),
"Hello Ada count=2 enabled=true tone=warm",
),
(
"greeting",
json!({"name":"Ada","count":7,"enabled":false,"tone":"dry"}),
"Hello Ada count=7 enabled=false tone=dry",
),
(
"archived-prompt",
json!({"name":"Ada"}),
"Hello Ada count=2 enabled=true tone=warm",
),
] {
let mut previous = None;
for _ in 0..2 {
let (result, delta) = success(
&h,
revision,
session,
"prompts/get",
json!({"name":name,"arguments":args}),
sse,
)
.await;
assert_eq!(result["description"], format!("Published {name}"));
assert_eq!(
result["messages"],
json!([
{"role":"system","content":{"type":"text","text":"Be brief."}},
{"role":"user","content":{"type":"text","text":text}}
])
);
if let Some(ref previous) = previous {
assert_eq!(&result, previous);
}
previous = Some(result);
deltas.push((delta, [1, 0, 1]));
}
}
for (arguments, reason, argument) in [
(json!({}), "missing_argument", "name"),
(
json!({"name":"Ada","extra":"x"}),
"unknown_argument",
"extra",
),
(json!({"name":5}), "argument_invalid", "name"),
(
json!({"name":"Ada","count":"7"}),
"argument_invalid",
"count",
),
(
json!({"name":"Ada","enabled":"false"}),
"argument_invalid",
"enabled",
),
(
json!({"name":"Ada","tone":"loud"}),
"argument_invalid",
"tone",
),
(
json!({"name":"x".repeat(crank_core::prompt::MAX_TEXT_BYTES + 1)}),
"argument_too_large",
"name",
),
] {
let req = request(
revision,
"prompts/get",
json!({"name":"greeting","arguments":arguments}),
);
let body = wire(h.post(revision, session, &req, sse).await, &req["id"], sse).await;
assert_eq!(body["error"]["code"], -32602, "{body}");
assert_eq!(body["error"]["data"]["reason"], reason, "{body}");
assert_eq!(body["error"]["data"]["argument"], argument);
assert!(body.get("result").is_none(), "{body}");
for leaked in [
"Be brief.",
"Hello ",
"Changed ",
"Published unbound-prompt",
] {
assert!(!body.to_string().contains(leaked), "{body}");
}
}
let mut unavailable = None;
for name in [
"draft-prompt",
"unbound-prompt",
"other-agent-prompt",
"never-existed",
"prm_a",
] {
let req = request(revision, "prompts/get", json!({"name":name}));
let body = wire(h.post(revision, session, &req, sse).await, &req["id"], sse).await;
assert_eq!(body["error"]["code"], -32602, "{body}");
assert_eq!(
body["error"]["message"],
"the requested prompt is not available"
);
assert!(body.get("result").is_none());
if let Some(ref previous) = unavailable {
assert_eq!(&body, previous);
}
unavailable = Some(body);
}
}
for params in [json!({"cursor":"invalid"}), json!({"limit":"invalid"})] {
let req = request(revision, "prompts/list", params);
let body = wire(
h.post(revision, session, &req, false).await,
&req["id"],
false,
)
.await;
assert_eq!(body["error"]["code"], -32602, "{body}");
}
let other = if revision == MODERN_REVISION {
LEGACY_REVISION
} else {
MODERN_REVISION
};
let other_session = h.session(other).await;
let req = request(
other,
"prompts/list",
json!({"cursor":first_cursor.unwrap()}),
);
let body = wire(
h.post(other, other_session.as_deref(), &req, false).await,
&req["id"],
false,
)
.await;
assert_eq!(
body["error"]["code"], -32602,
"cross-revision cursor: {body}"
);
// Archive keeps serving v1 above, but disallows new publications and bindings.
use crank_core::ids::PromptId;
use crank_registry::model::{
AgentPromptBinding, PublishPromptRequest, SaveAgentCatalogConfigRequest,
};
let publication = h
.registry
.publish_prompt_draft(PublishPromptRequest {
workspace_id: workspace_id(),
prompt_id: PromptId::new("prm_b"),
expected_updated_at: h.archived_at,
published_at: time::OffsetDateTime::now_utc(),
})
.await;
assert!(
matches!(
publication,
Err(crank_registry::RegistryError::PromptNotDraft { .. })
),
"{publication:?}"
);
let binding = h
.registry
.save_agent_catalog_config_editable_with_prompts(
SaveAgentCatalogConfigRequest {
workspace_id: &workspace_id(),
agent_id: &agent_id(),
agent_version: 1,
bindings: &[],
tool_selection_policy: &crank_core::ToolSelectionPolicy::default(),
resource_bindings: &[],
expected_state: None,
},
&[AgentPromptBinding {
prompt_id: PromptId::new("prm_b"),
prompt_version: 1,
}],
)
.await;
assert!(
matches!(
binding,
Err(crank_registry::RegistryError::PromptArchived { .. })
),
"{binding:?}"
);
h.registry
.revoke_platform_api_key(
&workspace_id(),
&crank_core::PlatformApiKeyId::new("pk_dispatch_observer"),
&time::OffsetDateTime::now_utc(),
)
.await
.unwrap();
for (method, params) in [
("prompts/list", json!({})),
(
"prompts/get",
json!({"name":"greeting","arguments":{"name":"Ada"}}),
),
] {
assert_eq!(
h.post(revision, session, &request(revision, method, params), false)
.await
.status(),
401
);
}
eprintln!(
"existing GREEN Prompts wire {revision}: populated pagination, typed metadata/values, exact immutable rendering, archive, structured refusals/non-disclosure, correlation/ID/cache, revoked401; SDK actual/expected={deltas:?}"
);
deltas
}
#[tokio::test]
async fn rmcp_prompts_wire_legacy() {
control(LEGACY_REVISION).await;
}
#[tokio::test]
async fn rmcp_prompts_wire_modern() {
control(MODERN_REVISION).await;
}
#[tokio::test]
async fn rmcp_prompts_sdk_legacy() {
let deltas = control(LEGACY_REVISION).await;
assert!(
deltas.iter().all(|(a, e)| a == e),
"SDK typed dispatch/result missing after successful HTTP: {deltas:?}"
);
}
#[tokio::test]
async fn rmcp_prompts_sdk_modern() {
let deltas = control(MODERN_REVISION).await;
assert!(
deltas.iter().all(|(a, e)| a == e),
"SDK typed dispatch/result missing after successful HTTP: {deltas:?}"
);
}
#[tokio::test]
async fn rmcp_prompts_observer_scope_control() {
use super::{
sdk_prompts_dispatch_enter, sdk_prompts_get_result_serialized,
sdk_prompts_list_result_serialized,
};
let a = Arc::new(CounterSet::default());
let b = Arc::new(CounterSet::default());
sdk_prompts_dispatch_enter();
assert_eq!(counts(&a), [0; 3]);
let snapshot = a.snapshot();
observe(a.clone(), async {
sdk_prompts_dispatch_enter();
sdk_prompts_list_result_serialized();
sdk_prompts_get_result_serialized();
})
.await;
observe(b.clone(), async {
sdk_prompts_get_result_serialized();
})
.await;
assert_eq!(counts(&a), [1; 3]);
assert_eq!(counts(&b), [0, 0, 1]);
assert_eq!(a.snapshot(), snapshot);
let body = Body::from_stream(stream::once(async {
sdk_prompts_list_result_serialized();
Ok::<_, std::convert::Infallible>("terminal")
}));
let response = observe_body(Response::new(body), b.clone());
assert_eq!(
&axum::body::to_bytes(response.into_body(), 100)
.await
.unwrap()[..],
b"terminal"
);
assert_eq!(counts(&b), [0, 1, 1]);
assert_eq!(counts(&a), [1; 3]);
}
@@ -0,0 +1,305 @@
//! Published Prompts fixture using the existing observed HTTP app and disposable PostgreSQL.
use super::*;
use crank_core::prompt::{PromptDefinition, PromptSensitivity};
use crank_core::{Agent, AgentStatus, AgentVersion, ToolSelectionPolicy, ids::PromptId};
use crank_registry::model::{
AgentPromptBinding, ArchivePromptRequest, CreatePromptDraftRequest, PublishPromptRequest,
SaveAgentCatalogConfigRequest, UpdatePromptDraftRequest,
};
use crank_registry::{CreateAgentRequest, PublishAgentRequest};
use crank_runtime::{
InMemoryCoordinationStateStore, RequestRateLimitConfig, RequestRateLimiter, SecretCrypto,
};
pub(super) fn definition(name: &str, content: &str) -> PromptDefinition {
PromptDefinition::parse(&json!({
"name":name, "description":format!("Published {name}"),
"arguments":[
{"name":"name","schema":{"type":"string"},"required":true},
{"name":"count","schema":{"type":"integer"},"required":false,"default":2},
{"name":"enabled","schema":{"type":"boolean"},"required":false,"default":true},
{"name":"tone","schema":{"type":"string","enum":["warm","dry"]},"required":false,"default":"warm"}
],
"messages":[{"role":"system","content":"Be brief."},{"role":"user","content":content}]
})).unwrap()
}
pub(super) struct Endpoint {
pub registry: crank_registry::PostgresRegistry,
pub counters: Arc<CounterSet>,
pub key: String,
pub url: String,
pub archived_at: time::OffsetDateTime,
server: tokio::task::JoinHandle<()>,
}
impl Drop for Endpoint {
fn drop(&mut self) {
self.server.abort();
}
}
impl Endpoint {
pub async fn started() -> Self {
crank_test_support::postgres_schema_url_for_task("prompts_owner", "t_160a763a").await;
let registry = test_registry().await;
let mut archived_at = test_time();
for (id, name, publish) in [
("prm_a", "greeting", true),
("prm_b", "archived-prompt", true),
("prm_c", "unbound-prompt", true),
("prm_d", "draft-prompt", false),
("prm_e", "other-agent-prompt", true),
] {
let row = registry
.create_prompt_draft(CreatePromptDraftRequest {
workspace_id: workspace_id(),
prompt_id: PromptId::new(id),
sensitivity: PromptSensitivity::Internal,
definition: definition(
name,
"Hello {{name}} count={{count}} enabled={{enabled}} tone={{tone}}",
),
created_at: time::OffsetDateTime::now_utc(),
})
.await
.unwrap();
if publish {
let row = registry
.publish_prompt_draft(PublishPromptRequest {
workspace_id: workspace_id(),
prompt_id: PromptId::new(id),
expected_updated_at: row.updated_at,
published_at: time::OffsetDateTime::now_utc(),
})
.await
.unwrap();
if id == "prm_b" {
archived_at = row.updated_at;
}
}
}
for (id, slug) in [
(agent_id(), AGENT_SLUG),
(
crank_core::AgentId::new("agent_other_prompts"),
"other-prompts",
),
] {
registry
.create_agent(CreateAgentRequest {
agent: &Agent {
id: id.clone(),
workspace_id: workspace_id(),
slug: slug.into(),
display_name: "SDK Prompts".into(),
description: String::new(),
status: AgentStatus::Draft,
current_draft_version: 1,
latest_published_version: None,
created_at: test_time(),
updated_at: test_time(),
published_at: None,
},
version: &AgentVersion {
agent_id: id,
version: 1,
status: AgentStatus::Draft,
instructions: json!({}),
tool_selection_policy: ToolSelectionPolicy::default(),
created_at: test_time(),
},
bindings: &[],
})
.await
.unwrap();
}
let other_agent = crank_core::AgentId::new("agent_other_prompts");
registry
.save_agent_catalog_config_editable_with_prompts(
SaveAgentCatalogConfigRequest {
workspace_id: &workspace_id(),
agent_id: &other_agent,
agent_version: 1,
bindings: &[],
tool_selection_policy: &ToolSelectionPolicy::default(),
resource_bindings: &[],
expected_state: None,
},
&[AgentPromptBinding {
prompt_id: PromptId::new("prm_e"),
prompt_version: 1,
}],
)
.await
.unwrap();
registry
.publish_agent(PublishAgentRequest {
workspace_id: &workspace_id(),
agent_id: &other_agent,
version: 1,
published_at: &test_time(),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
registry
.save_agent_catalog_config_editable_with_prompts(
SaveAgentCatalogConfigRequest {
workspace_id: &workspace_id(),
agent_id: &agent_id(),
agent_version: 1,
bindings: &[],
tool_selection_policy: &ToolSelectionPolicy::default(),
resource_bindings: &[],
expected_state: None,
},
&[
AgentPromptBinding {
prompt_id: PromptId::new("prm_a"),
prompt_version: 1,
},
AgentPromptBinding {
prompt_id: PromptId::new("prm_b"),
prompt_version: 1,
},
],
)
.await
.unwrap();
registry
.publish_agent(PublishAgentRequest {
workspace_id: &workspace_id(),
agent_id: &agent_id(),
version: 1,
published_at: &test_time(),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
let archived_at = registry
.archive_prompt_draft(ArchivePromptRequest {
workspace_id: workspace_id(),
prompt_id: PromptId::new("prm_b"),
expected_updated_at: archived_at,
archived_at: time::OffsetDateTime::now_utc(),
})
.await
.unwrap()
.updated_at;
// A mutable v2 draft must not change the bound v1 rendering or discovery metadata.
let draft = registry
.get_prompt_draft(&workspace_id(), &PromptId::new("prm_a"))
.await
.unwrap();
registry
.update_prompt_draft(UpdatePromptDraftRequest {
workspace_id: workspace_id(),
prompt_id: PromptId::new("prm_a"),
sensitivity: PromptSensitivity::Internal,
definition: definition("changed-draft", "Changed {{name}}"),
expected_updated_at: draft.updated_at,
updated_at: time::OffsetDateTime::now_utc(),
})
.await
.unwrap();
let key = create_api_key(&registry).await;
let counters = Arc::new(CounterSet::default());
let app = crate::build_app(
registry.clone(),
Duration::ZERO,
None,
SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(),
crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build(),
RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()),
Arc::new(InMemoryCoordinationStateStore::default()),
Arc::new(crate::session::InMemorySessionStore::default()),
Arc::new(crate::auth::CommunityMachineCredentialVerifier),
);
let app = observe_requests(app, counters.clone(), None).layer(axum::middleware::from_fn({
let counters = counters.clone();
move |request: axum::extract::Request, next: axum::middleware::Next| {
let counters = counters.clone();
async move { observe_body(next.run(request).await, counters) }
}
}));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!(
"http://{}/v1/{WORKSPACE_SLUG}/{AGENT_SLUG}",
listener.local_addr().unwrap()
);
let server = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
Self {
registry,
counters,
key,
url,
archived_at,
server,
}
}
pub async fn post(
&self,
revision: &str,
session: Option<&str>,
body: &Value,
sse: bool,
) -> reqwest::Response {
let mut request = reqwest::Client::new()
.post(&self.url)
.bearer_auth(&self.key)
.header(
"Accept",
if sse {
"text/event-stream, application/json"
} else {
"application/json"
},
)
.header("x-request-id", "sdk-prompts-wire")
.header(
"traceparent",
"00-11111111111111111111111111111111-2222222222222222-01",
)
.header("MCP-Protocol-Version", revision);
if let Some(session) = session {
request = request.header("MCP-Session-Id", session);
}
if revision == MODERN_REVISION {
request = request.header("Mcp-Method", body["method"].as_str().unwrap());
if let Some(name) = body["params"]["name"].as_str() {
request = request.header("Mcp-Name", name);
}
}
request.json(body).send().await.unwrap()
}
pub async fn session(&self, revision: &str) -> Option<String> {
if revision == MODERN_REVISION {
return None;
}
let response = self
.post(revision, None, &legacy_initialize(1), false)
.await;
assert_eq!(response.status(), 200);
let session = response.headers()["MCP-Session-Id"]
.to_str()
.unwrap()
.to_owned();
assert_eq!(
json_body(response).await["result"]["protocolVersion"],
revision
);
assert_eq!(
self.post(revision, Some(&session), &initialized_notification(), false)
.await
.status(),
202
);
Some(session)
}
}
@@ -0,0 +1,287 @@
//! #107 Resources: existing wire GREEN is independent of the missing SDK typed-path oracle.
use super::{CounterSet, observe, support::*};
use axum::{body::Body, response::Response};
use futures_util::{StreamExt, stream};
use serde_json::{Value, json};
use std::{sync::Arc, time::Duration};
#[path = "rmcp_resources_test_support.rs"]
mod fixtures;
use fixtures::{Endpoint, TEXT, URI};
// As in rmcp_result_runtime_tests: deferred body polling keeps this request's observer.
fn observe_body(response: Response, counters: Arc<CounterSet>) -> Response {
let (parts, body) = response.into_parts();
let body = stream::unfold(
(body.into_data_stream(), counters),
|(mut body, counters)| async move {
observe(counters.clone(), body.next())
.await
.map(|chunk| (chunk, (body, counters)))
},
);
Response::from_parts(parts, Body::from_stream(body))
}
fn counts(c: &CounterSet) -> [usize; 4] {
[
c.sdk_resources_dispatch(),
c.sdk_resources_list_result(),
c.sdk_resources_read_result(),
c.sdk_resource_templates_list_result(),
]
}
fn request(revision: &str, method: &str, mut params: Value) -> Value {
if revision == MODERN_REVISION {
params["_meta"] = modern_meta();
}
json!({"jsonrpc":"2.0", "id":9007199254740993_u64, "method":method, "params":params})
}
async fn wire(response: reqwest::Response, id: &Value, sse: bool) -> Value {
assert_eq!(response.status(), 200);
assert_eq!(response.headers()["x-request-id"], "sdk-resources-wire");
assert_eq!(
response.headers()["x-trace-id"],
"11111111111111111111111111111111"
);
let content_type = response.headers()["content-type"]
.to_str()
.unwrap()
.to_owned();
let text = tokio::time::timeout(Duration::from_secs(5), response.text())
.await
.unwrap()
.unwrap();
let body: Value = if sse {
assert!(
content_type.starts_with("text/event-stream"),
"{content_type}: {text}"
);
let frames: Vec<Value> = text
.lines()
.filter_map(|line| {
line.strip_prefix("data:")
.map(|s| serde_json::from_str(s.trim()).unwrap())
})
.collect();
let terminal: Vec<_> = frames.iter().filter(|f| f.get("id") == Some(id)).collect();
assert_eq!(terminal.len(), 1, "{frames:?}");
assert_eq!(frames.last(), Some(terminal[0]), "terminal must be last");
terminal[0].clone()
} else {
assert!(
content_type.starts_with("application/json"),
"{content_type}: {text}"
);
serde_json::from_str(&text).unwrap()
};
assert_eq!(body["jsonrpc"], "2.0");
assert_eq!(&body["id"], id);
body
}
fn era(result: &Value, revision: &str) {
if revision == MODERN_REVISION {
assert_eq!(result["resultType"], "complete", "{result}");
assert_eq!(result["ttlMs"], 0);
assert_eq!(result["cacheScope"], "private");
} else {
for key in ["resultType", "ttlMs", "cacheScope"] {
assert!(result.get(key).is_none(), "{result}");
}
}
assert!(result.get("_meta").is_none(), "{result}");
}
async fn success(
h: &Endpoint,
revision: &str,
session: Option<&str>,
method: &str,
params: Value,
sse_accept: bool,
) -> (Value, [usize; 4]) {
let req = request(revision, method, params);
let before = counts(&h.counters);
let body = wire(
h.post(revision, session, &req, sse_accept).await,
&req["id"],
sse_accept,
)
.await;
assert!(body.get("error").is_none(), "{body}");
era(&body["result"], revision);
let after = counts(&h.counters);
(
body["result"].clone(),
std::array::from_fn(|i| after[i] - before[i]),
)
}
async fn control(revision: &str) -> Vec<([usize; 4], [usize; 4])> {
let h = Endpoint::started().await;
let session = h.session(revision).await;
let session = session.as_deref();
let mut deltas = Vec::new();
// Preference order negotiates JSON or terminal SSE in both Resources revisions.
for sse in [false, true] {
let mut cursor = None;
let mut names = Vec::new();
for page in 0..2 {
let mut params = json!({"limit":1});
if let Some(ref cursor) = cursor {
params["cursor"] = json!(cursor);
}
let (result, delta) =
success(&h, revision, session, "resources/list", params, sse).await;
let resources = result["resources"].as_array().unwrap();
assert_eq!(resources.len(), 1, "{result}");
let item = &resources[0];
assert_eq!(item["mimeType"], "text/plain");
assert!(
item.get("description").is_none(),
"static descriptors omit description: {item}"
);
names.push((item["name"].clone(), item["uri"].clone()));
cursor = result
.get("nextCursor")
.map(|v| v.as_str().unwrap().to_owned());
assert_eq!(cursor.is_some(), page == 0, "{result}");
if let Some(ref cursor) = cursor {
assert_ne!(cursor, "1", "opaque cursor required");
}
deltas.push((delta, [1, 1, 0, 0]));
}
names.sort_by_key(|n| n.0.as_str().unwrap().to_owned());
assert_eq!(
names,
vec![
(json!("sdk_a"), json!(URI)),
(json!("sdk_b"), json!("crank://resources/sdk-b"))
]
);
let (result, delta) = success(
&h,
revision,
session,
"resources/read",
json!({"uri":URI}),
sse,
)
.await;
assert_eq!(
result["contents"],
json!([{"uri":URI,"mimeType":"text/plain","text":TEXT}])
);
deltas.push((delta, [1, 0, 1, 0]));
let (result, delta) = success(
&h,
revision,
session,
"resources/templates/list",
json!({}),
sse,
)
.await;
assert_eq!(result["resourceTemplates"], json!([]));
assert!(result.get("nextCursor").is_none());
deltas.push((delta, [1, 0, 0, 1]));
}
// Malformed parameters/cursors and indistinguishable unavailable URIs remain command errors.
for (method, params) in [
("resources/list", json!({"cursor":"invalid"})),
("resources/list", json!({"limit":"invalid"})),
("resources/templates/list", json!({"cursor":"invalid"})),
("resources/templates/list", json!({"limit":"invalid"})),
("resources/read", json!({"uri":"crank://resources/unbound"})),
] {
let req = request(revision, method, params);
let body = wire(
h.post(revision, session, &req, false).await,
&req["id"],
false,
)
.await;
assert_eq!(body["error"]["code"], -32602, "{body}");
assert!(body.get("result").is_none(), "{body}");
}
// An existing session does not preserve revoked authorization.
h.registry
.revoke_platform_api_key(
&workspace_id(),
&crank_core::PlatformApiKeyId::new("pk_dispatch_observer"),
&time::OffsetDateTime::now_utc(),
)
.await
.unwrap();
for (method, params) in [
("resources/list", json!({})),
("resources/read", json!({"uri":URI})),
("resources/templates/list", json!({})),
] {
let req = request(revision, method, params);
assert_eq!(h.post(revision, session, &req, false).await.status(), 401);
}
eprintln!(
"existing GREEN Resources wire {revision}: pagination, exact content URI/bytes, empty templates, JSON/SSE negotiation, InvalidParams, revoked401; actual/expected SDK deltas={deltas:?}"
);
deltas
}
#[tokio::test]
async fn rmcp_resources_wire_legacy() {
control(LEGACY_REVISION).await;
}
#[tokio::test]
async fn rmcp_resources_wire_modern() {
control(MODERN_REVISION).await;
}
#[tokio::test]
async fn rmcp_resources_sdk_legacy() {
let deltas = control(LEGACY_REVISION).await;
assert!(
deltas.iter().all(|(actual, expected)| actual == expected),
"SDK typed dispatch/result missing after successful HTTP: {deltas:?}"
);
}
#[tokio::test]
async fn rmcp_resources_sdk_modern() {
let deltas = control(MODERN_REVISION).await;
assert!(
deltas.iter().all(|(actual, expected)| actual == expected),
"SDK typed dispatch/result missing after successful HTTP: {deltas:?}"
);
}
#[tokio::test]
async fn rmcp_resources_observer_scope_control() {
use super::{
sdk_resource_templates_list_result_serialized, sdk_resources_dispatch_enter,
sdk_resources_list_result_serialized, sdk_resources_read_result_serialized,
};
let a = Arc::new(CounterSet::default());
let b = Arc::new(CounterSet::default());
sdk_resources_dispatch_enter();
assert_eq!(counts(&a), [0; 4]);
let snapshot = a.snapshot();
observe(a.clone(), async {
sdk_resources_dispatch_enter();
sdk_resources_list_result_serialized();
sdk_resources_read_result_serialized();
sdk_resource_templates_list_result_serialized();
})
.await;
observe(b.clone(), async {
sdk_resources_read_result_serialized();
})
.await;
assert_eq!(counts(&a), [1; 4]);
assert_eq!(counts(&b), [0, 0, 1, 0]);
assert_eq!(a.snapshot(), snapshot, "old Snapshot stays unchanged");
let body = Body::from_stream(stream::once(async {
sdk_resource_templates_list_result_serialized();
Ok::<_, std::convert::Infallible>("terminal")
}));
let response = observe_body(Response::new(body), b.clone());
assert_eq!(
&axum::body::to_bytes(response.into_body(), 100)
.await
.unwrap()[..],
b"terminal"
);
assert_eq!(counts(&b), [0, 0, 1, 1]);
assert_eq!(counts(&a), [1; 4]);
}
@@ -0,0 +1,227 @@
//! Small Resources fixture: existing observer helpers, exact artifact publications, normal app.
use super::*;
use crank_artifacts::ArtifactStore;
use crank_core::{Agent, AgentStatus, AgentVersion, ToolSelectionPolicy, ids::ResourceId};
use crank_registry::{
AgentResourceBinding, CreateAgentRequest, PublishAgentRequest, SaveAgentCatalogConfigRequest,
};
use crank_runtime::{
InMemoryCoordinationStateStore, RequestRateLimitConfig, RequestRateLimiter, SecretCrypto,
};
use sha2::{Digest, Sha256};
pub(super) const URI: &str = "crank://resources/sdk-a";
pub(super) const TEXT: &str = "published v1\n";
pub(super) struct Endpoint {
pub registry: crank_registry::PostgresRegistry,
pub counters: Arc<CounterSet>,
pub key: String,
pub url: String,
server: tokio::task::JoinHandle<()>,
root: std::path::PathBuf,
}
impl Drop for Endpoint {
fn drop(&mut self) {
self.server.abort();
let _ = std::fs::remove_dir_all(&self.root);
}
}
impl Endpoint {
pub async fn started() -> Self {
crank_test_support::postgres_schema_url_for_task("resources_owner", "t_73561633").await;
let registry = test_registry().await;
let root = std::env::temp_dir().join(format!("rmcp-resources-{}", uuid::Uuid::now_v7()));
std::fs::create_dir(&root).unwrap();
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap();
let store = ArtifactStore::open(&root).unwrap();
// Same immutable row shapes and real ArtifactStore::put as static_resources/fixtures.rs.
for (id, uri, name, bytes) in [
("res_sdk_a", URI, "sdk_a", TEXT),
(
"res_sdk_b",
"crank://resources/sdk-b",
"sdk_b",
"second published resource\n",
),
] {
let stored = store.put(bytes.as_bytes()).unwrap();
let digest = format!("{:x}", Sha256::digest(bytes.as_bytes()));
let source = format!("src_{id}");
sqlx::query("insert into artifact_blobs (digest, artifact_ref, size_bytes, storage_lifecycle, created_at, updated_at) values ($1,$2,$3,'available',now(),now())")
.bind(&digest).bind(format!("sha256:{digest}")).bind(stored.size_bytes as i64).execute(registry.pool()).await.unwrap();
sqlx::query("insert into artifact_sources (workspace_id,source_id,blob_digest,mime_type,sensitivity,lifecycle,created_at,updated_at) values ('ws_default',$1,$2,'text/plain','internal','active',now(),now())")
.bind(&source).bind(&digest).execute(registry.pool()).await.unwrap();
sqlx::query("insert into resources (workspace_id,resource_id,uri,name,description,status,kind,current_draft_version,created_at,updated_at) values ('ws_default',$1,$2,$3,'','published','text',1,now(),now())")
.bind(id).bind(uri).bind(name).execute(registry.pool()).await.unwrap();
sqlx::query("insert into resource_versions (workspace_id,resource_id,version,kind,mime_type,size_bytes,digest,sensitivity,artifact_source_id,created_at,superseded_at) values ('ws_default',$1,1,'text','text/plain',$2,$3,'internal',$4,now(),null)")
.bind(id).bind(stored.size_bytes as i64).bind(&digest).bind(&source).execute(registry.pool()).await.unwrap();
sqlx::query("insert into resource_publications (workspace_id,resource_id,version,uri,name,description,kind,mime_type,size_bytes,digest,sensitivity,artifact_source_id,published_at) values ('ws_default',$1,1,$2,$3,'','text','text/plain',$4,$5,'internal',$6,now())")
.bind(id).bind(uri).bind(name).bind(stored.size_bytes as i64).bind(&digest).bind(&source).execute(registry.pool()).await.unwrap();
}
registry
.create_agent(CreateAgentRequest {
agent: &Agent {
id: agent_id(),
workspace_id: workspace_id(),
slug: AGENT_SLUG.into(),
display_name: "SDK Resources".into(),
description: String::new(),
status: AgentStatus::Draft,
current_draft_version: 1,
latest_published_version: None,
created_at: test_time(),
updated_at: test_time(),
published_at: None,
},
version: &AgentVersion {
agent_id: agent_id(),
version: 1,
status: AgentStatus::Draft,
instructions: json!({}),
tool_selection_policy: ToolSelectionPolicy::default(),
created_at: test_time(),
},
bindings: &[],
})
.await
.unwrap();
registry
.save_agent_catalog_config(SaveAgentCatalogConfigRequest {
workspace_id: &workspace_id(),
agent_id: &agent_id(),
agent_version: 1,
bindings: &[],
tool_selection_policy: &ToolSelectionPolicy::default(),
resource_bindings: &[
AgentResourceBinding {
resource_id: ResourceId::new("res_sdk_a"),
resource_version: 1,
},
AgentResourceBinding {
resource_id: ResourceId::new("res_sdk_b"),
resource_version: 1,
},
],
expected_state: None,
})
.await
.unwrap();
registry
.publish_agent(PublishAgentRequest {
workspace_id: &workspace_id(),
agent_id: &agent_id(),
version: 1,
published_at: &test_time(),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
let key = create_api_key(&registry).await;
let counters = Arc::new(CounterSet::default());
// Normal build_app composition with its sanctioned artifact root, not the isolated SDK router.
let app = crate::build_app_with_artifacts(
registry.clone(),
Duration::ZERO,
None,
SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(),
crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build(),
RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()),
Arc::new(InMemoryCoordinationStateStore::default()),
Arc::new(crate::session::InMemorySessionStore::default()),
Arc::new(crate::auth::CommunityMachineCredentialVerifier),
16,
store,
);
let app = observe_requests(app, counters.clone(), None).layer(axum::middleware::from_fn({
let counters = counters.clone();
move |request: axum::extract::Request, next: axum::middleware::Next| {
let counters = counters.clone();
async move {
let response = next.run(request).await;
observe_body(response, counters)
}
}
}));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!(
"http://{}/v1/{WORKSPACE_SLUG}/{AGENT_SLUG}",
listener.local_addr().unwrap()
);
let server = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
Self {
registry,
counters,
key,
url,
server,
root,
}
}
pub async fn post(
&self,
revision: &str,
session: Option<&str>,
body: &Value,
sse: bool,
) -> reqwest::Response {
let mut request = reqwest::Client::new()
.post(&self.url)
.bearer_auth(&self.key)
.header(
"Accept",
if sse {
"text/event-stream, application/json"
} else {
"application/json"
},
)
.header("x-request-id", "sdk-resources-wire")
.header(
"traceparent",
"00-11111111111111111111111111111111-2222222222222222-01",
)
.header("MCP-Protocol-Version", revision);
if let Some(session) = session {
request = request.header("MCP-Session-Id", session);
}
if revision == MODERN_REVISION {
request = request.header("Mcp-Method", body["method"].as_str().unwrap());
if let Some(uri) = body["params"]["uri"].as_str() {
request = request.header("Mcp-Name", uri);
}
}
request.json(body).send().await.unwrap()
}
pub async fn session(&self, revision: &str) -> Option<String> {
if revision == MODERN_REVISION {
return None;
}
let response = self
.post(revision, None, &legacy_initialize(1), false)
.await;
assert_eq!(response.status(), 200);
let session = response.headers()["MCP-Session-Id"]
.to_str()
.unwrap()
.to_owned();
assert_eq!(
json_body(response).await["result"]["protocolVersion"],
revision
);
assert_eq!(
self.post(revision, Some(&session), &initialized_notification(), false)
.await
.status(),
202
);
Some(session)
}
}
@@ -0,0 +1,437 @@
//! #107 result serialization: wire controls run independently of the structural SDK oracle.
use super::*;
use axum::{
Json,
body::Body,
extract::State,
http::StatusCode,
response::{IntoResponse, Response},
};
use futures_util::{StreamExt, stream};
const ERROR_EMAIL: &str = "upstream-error@example.test";
const REQUEST_ID: &str = "sdk-result-wire";
const TRACE_ID: &str = "11111111111111111111111111111111";
pub(super) async fn upstream(state: State<Arc<AtomicUsize>>, payload: Json<Value>) -> Response {
if payload.0["email"] == ERROR_EMAIL {
state.0.fetch_add(1, Ordering::SeqCst);
return (
StatusCode::UNPROCESSABLE_ENTITY,
Json(json!({"error":"fixture rejection"})),
)
.into_response();
}
create_lead(state, payload).await.into_response()
}
// The HTTP body is polled after next.run returns. Carry only the test observer, not a new lifecycle.
pub(super) fn observe_body(response: Response, counters: Arc<CounterSet>) -> Response {
let (parts, body) = response.into_parts();
let body = stream::unfold(
(body.into_data_stream(), counters),
|(mut body, counters)| async move {
observe(counters.clone(), body.next())
.await
.map(|chunk| (chunk, (body, counters)))
},
);
Response::from_parts(parts, Body::from_stream(body))
}
#[tokio::test]
async fn rmcp_result_http_pagination_both_revisions() {
let h = Endpoint::started().await;
let mut operation = test_operation(&h._upstream.1);
operation.id = crank_core::OperationId::new("op_second_lead");
operation.name = "second_lead".to_owned();
h.registry
.create_operation(&workspace_id(), &operation, Some("alice"))
.await
.unwrap();
h.registry
.publish_operation(PublishRequest {
workspace_id: &workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &test_time(),
published_by: Some("alice"),
})
.await
.unwrap();
let now = time::OffsetDateTime::now_utc();
h.registry
.create_agent_draft_version(CreateAgentDraftVersionRequest {
workspace_id: &workspace_id(),
agent_id: &agent_id(),
version: &AgentVersion {
agent_id: agent_id(),
version: 2,
status: AgentStatus::Draft,
instructions: json!({}),
tool_selection_policy: ToolSelectionPolicy::default(),
created_at: now,
},
bindings: &[
AgentOperationBinding {
operation_id: crank_core::OperationId::new("op_create_lead"),
operation_version: 1,
tool_name: TOOL_NAME.into(),
tool_title: "Create Lead".into(),
tool_description_override: None,
enabled: true,
},
AgentOperationBinding {
operation_id: operation.id,
operation_version: 1,
tool_name: "second_lead".into(),
tool_title: "Second Lead".into(),
tool_description_override: None,
enabled: true,
},
],
updated_at: &now,
})
.await
.unwrap();
h.registry
.publish_agent(PublishAgentRequest {
workspace_id: &workspace_id(),
agent_id: &agent_id(),
version: 2,
published_at: &now,
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
for revision in [LEGACY_REVISION, MODERN_REVISION] {
let session = h.session(revision).await;
let mut names = Vec::new();
for (cursor, expected_next, expected_len) in [
(None, Some("1"), 1),
(Some("1"), None, 1),
(Some("2"), None, 0),
] {
let id = json!(format!("page-{cursor:?}"));
let mut request =
json!({"jsonrpc":"2.0","id":id,"method":"tools/list","params":{"limit":1}});
if let Some(cursor) = cursor {
request["params"]["cursor"] = json!(cursor);
}
if revision == MODERN_REVISION {
request["params"]["_meta"] = modern_meta();
}
let before = h.counters.sdk_tools_list_result();
let body = wire_body(
h.post(revision, session.as_deref(), &request).await,
&id,
false,
)
.await;
assert_era(&body["result"], revision, true);
assert_eq!(
body["result"].get("nextCursor"),
expected_next.map(|s| json!(s)).as_ref()
);
let tools = body["result"]["tools"].as_array().unwrap();
assert_eq!(tools.len(), expected_len);
names.extend(
tools
.iter()
.map(|tool| tool["name"].as_str().unwrap().to_owned()),
);
assert_eq!(h.counters.sdk_tools_list_result() - before, 1);
}
names.sort();
let mut expected = vec![TOOL_NAME.to_owned(), "second_lead".to_owned()];
expected.sort();
assert_eq!(names, expected);
}
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
}
fn assert_era(result: &Value, revision: &str, list: bool) {
if revision == MODERN_REVISION {
assert_eq!(result["resultType"], "complete", "{result}");
if list {
assert_eq!(result["ttlMs"], 0);
assert_eq!(result["cacheScope"], "private");
}
} else {
assert!(result.get("resultType").is_none(), "{result}");
assert!(result.get("ttlMs").is_none(), "{result}");
assert!(result.get("cacheScope").is_none(), "{result}");
}
assert!(result.get("_meta").is_none(), "{result}");
}
async fn wire_body(response: reqwest::Response, id: &Value, sse: bool) -> Value {
assert_eq!(response.status(), 200);
assert_eq!(response.headers()["x-request-id"], REQUEST_ID);
assert_eq!(response.headers()["x-trace-id"], TRACE_ID);
let content_type = response.headers()["content-type"]
.to_str()
.unwrap()
.to_owned();
let text = tokio::time::timeout(Duration::from_secs(5), response.text())
.await
.unwrap()
.unwrap();
let body: Value = if sse {
assert!(
content_type.starts_with("text/event-stream"),
"{content_type}"
);
let frames: Vec<Value> = text
.lines()
.filter_map(|line| {
line.strip_prefix("data:")
.map(|data| serde_json::from_str(data.trim()).unwrap())
})
.collect();
let terminal: Vec<_> = frames
.iter()
.filter(|frame| frame.get("id") == Some(id))
.collect();
assert_eq!(terminal.len(), 1, "{frames:?}");
assert_eq!(frames.last(), Some(terminal[0]), "terminal must be last");
assert!(
frames
.iter()
.any(|frame| frame["method"] == "notifications/progress"
&& frame["params"]["progressToken"] == "result-progress"),
"{frames:?}"
);
terminal[0].clone()
} else {
assert!(
content_type.starts_with("application/json"),
"{content_type}"
);
serde_json::from_str(&text).unwrap()
};
assert_eq!(body["jsonrpc"], "2.0");
assert_eq!(&body["id"], id);
body
}
fn assert_call_result(body: &Value, revision: &str, email: &str) {
assert!(body.get("error").is_none(), "{body}");
let result = &body["result"];
assert_era(result, revision, false);
let (output, text, is_error) = if email == ERROR_EMAIL {
let contract = crate::tool_error::tool_error_contract_from_runtime(
&crank_runtime::RuntimeError::ProtocolAdapter(
crank_core::ProtocolAdapterError::UnexpectedStatus {
status: 422,
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
},
),
REQUEST_ID,
TRACE_ID,
crank_core::ExecutionLocale::En,
);
(
json!({"error":crate::tool_error::tool_error_value(&contract)}),
crate::tool_error::tool_error_text(&contract),
true,
)
} else {
let output = json!({"id":format!("lead_for_{email}")});
let text = serde_json::to_string_pretty(&output).unwrap();
(output, text, false)
};
assert_eq!(result["structuredContent"], output, "{body}");
assert_eq!(result["content"], json!([{"type":"text", "text":text}]));
assert_eq!(result["isError"], is_error);
assert_eq!(
result.as_object().unwrap().len(),
if revision == MODERN_REVISION { 4 } else { 3 },
"no unexpected optional fields: {result}"
);
}
async fn result_control(revision: &str) -> Vec<usize> {
let h = Endpoint::started().await;
let session = h.session(revision).await;
let session = session.as_deref();
let mut counts = Vec::new();
// Actual published descriptor and schema, then the empty page at the catalog boundary.
for (cursor, expected_tools, id) in [
(
None,
json!([{"name":TOOL_NAME,"title":"Create Lead","description":"Creates a CRM lead","inputSchema":{"type":"object","properties":{"email":{"type":"string"}},"required":["email"]}}]),
json!("list-string-id"),
),
(Some("1"), json!([]), json!(9007199254740993_u64)),
] {
let mut request =
json!({"jsonrpc":"2.0","id":id,"method":"tools/list","params":{"limit":1}});
if revision == MODERN_REVISION {
request["params"]["_meta"] = modern_meta();
}
if let Some(cursor) = cursor {
request["params"]["cursor"] = json!(cursor);
}
let before = h.counters.sdk_tools_list_result();
let before_call = h.counters.sdk_tools_call_result();
let before_dispatch = h.counters.sdk_tools_dispatch();
let body = wire_body(h.post(revision, session, &request).await, &id, false).await;
assert!(body.get("error").is_none(), "{body}");
assert_eq!(body["result"]["tools"], expected_tools);
assert!(body["result"].get("nextCursor").is_none(), "{body}");
assert_era(&body["result"], revision, true);
assert_eq!(h.counters.sdk_tools_call_result(), before_call);
assert_eq!(h.counters.sdk_tools_dispatch() - before_dispatch, 1);
counts.push(h.counters.sdk_tools_list_result() - before);
}
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
for (email, id, sse) in [
(
"result-success@example.test",
json!(9007199254740993_u64),
false,
),
(ERROR_EMAIL, json!("tool-error-string-id"), false),
("result-sse@example.test", json!("sse-string-id"), true),
(ERROR_EMAIL, json!(92), true),
] {
if sse && revision != MODERN_REVISION {
continue;
}
let mut request = if revision == MODERN_REVISION {
modern_tool_call(90, email)
} else {
legacy_tool_call(90, email)
};
request["id"] = id.clone();
if sse {
request["params"]["_meta"]["progressToken"] = json!("result-progress");
}
let before = h.counters.sdk_tools_call_result();
let before_list = h.counters.sdk_tools_list_result();
let before_dispatch = h.counters.sdk_tools_dispatch();
let arrivals = h.calls.load(Ordering::SeqCst);
let response = h
.post_accept(
revision,
session,
&request,
if sse {
"application/json, text/event-stream"
} else {
"application/json"
},
)
.await;
let body = wire_body(response, &id, sse).await;
assert_call_result(&body, revision, email);
assert_eq!(h.calls.load(Ordering::SeqCst) - arrivals, 1);
assert_eq!(h.counters.sdk_tools_list_result(), before_list);
assert_eq!(h.counters.sdk_tools_dispatch() - before_dispatch, 1);
counts.push(h.counters.sdk_tools_call_result() - before);
}
// Unknown tool is a command InvalidParams, not an execution tool-error result.
let mut invalid = if revision == MODERN_REVISION {
modern_tool_call(93, "unused@example.test")
} else {
legacy_tool_call(93, "unused@example.test")
};
invalid["params"]["name"] = json!("missing_tool");
invalid["id"] = json!("invalid-params-id");
let before = h.counters.sdk_tools_call_result();
let arrivals = h.calls.load(Ordering::SeqCst);
let body = wire_body(
h.post(revision, session, &invalid).await,
&invalid["id"],
false,
)
.await;
assert_eq!(body["error"]["code"], -32602, "{body}");
assert!(body.get("result").is_none(), "{body}");
assert_eq!(body["error"]["data"]["request_id"], REQUEST_ID);
assert_eq!(body["error"]["data"]["trace_id"], TRACE_ID);
assert_eq!(h.counters.sdk_tools_call_result(), before);
assert_eq!(h.calls.load(Ordering::SeqCst), arrivals);
eprintln!(
"existing GREEN result wire {revision}: published list/schema + empty page, exact success/error JSON{}; InvalidParams/no result/no marker/no outbound; SDK result deltas={counts:?}",
if revision == MODERN_REVISION {
" + success/error terminal SSE"
} else {
" (legacy JSON-only)"
}
);
counts
}
#[tokio::test]
async fn rmcp_result_wire_modern() {
result_control(MODERN_REVISION).await;
}
#[tokio::test]
async fn rmcp_result_wire_legacy() {
result_control(LEGACY_REVISION).await;
}
#[tokio::test]
async fn rmcp_result_sdk_serialization_modern() {
assert_eq!(
result_control(MODERN_REVISION).await,
vec![1; 6],
"structural oracle: SDK list/final-empty/success/error/SSE-success/SSE-error must each serialize once after successful real HTTP wire controls"
);
}
#[tokio::test]
async fn rmcp_result_sdk_serialization_legacy() {
assert_eq!(
result_control(LEGACY_REVISION).await,
vec![1; 4],
"structural oracle: SDK list/final-empty/success/error must each serialize once after successful real HTTP wire controls"
);
}
#[tokio::test]
async fn rmcp_result_observer_scope_control() {
use crate::app::dispatch_observer::{
sdk_tools_call_result_serialized, sdk_tools_list_result_serialized,
};
let a = Arc::new(CounterSet::default());
let b = Arc::new(CounterSet::default());
sdk_tools_list_result_serialized();
sdk_tools_call_result_serialized();
assert_eq!(
(a.sdk_tools_list_result(), a.sdk_tools_call_result()),
(0, 0)
);
observe(a.clone(), async {
sdk_tools_list_result_serialized();
sdk_tools_call_result_serialized();
})
.await;
// A different request cannot alter a's result counters.
observe(b.clone(), async {
sdk_tools_list_result_serialized();
})
.await;
assert_eq!(
(a.sdk_tools_list_result(), a.sdk_tools_call_result()),
(1, 1)
);
assert_eq!(
(b.sdk_tools_list_result(), b.sdk_tools_call_result()),
(1, 0)
);
// Prove the stream-body scope actually records deferred work after the header scope ended.
let stream_counters = b.clone();
let body = Body::from_stream(stream::once(async move {
sdk_tools_call_result_serialized();
Ok::<_, std::convert::Infallible>("terminal")
}));
let response = observe_body(Response::new(body), stream_counters);
let bytes = axum::body::to_bytes(response.into_body(), 100)
.await
.unwrap();
assert_eq!(&bytes[..], b"terminal");
assert_eq!(b.sdk_tools_call_result(), 1);
assert_eq!(a.sdk_tools_call_result(), 1);
}
@@ -0,0 +1,385 @@
//! #107: real build_app endpoint controls plus a separate SDK structural seam oracle.
//! No SDK transport router is used. The structural oracle alone does not prove wire parity.
use std::{
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use axum::{Router, routing::post};
use crank_core::{
AgentOperationBinding, AgentStatus, AgentVersion, PlatformApiKeyId, ToolSelectionPolicy,
};
use crank_registry::{
CreateAgentDraftVersionRequest, PostgresRegistry, PublishAgentRequest, PublishRequest,
};
use crank_runtime::{
InMemoryCoordinationStateStore, RequestRateLimitConfig, RequestRateLimiter, SecretCrypto,
};
use serde_json::{Value, json};
use super::{CounterSet, observe, sdk_tools_dispatch_enter, support::*};
#[path = "rmcp_result_runtime_tests.rs"]
mod result_serialization;
struct Server(tokio::task::JoinHandle<()>, String);
impl Drop for Server {
fn drop(&mut self) {
self.0.abort();
}
}
async fn serve(app: Router) -> Server {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
Server(
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
}),
format!("http://{address}"),
)
}
struct Endpoint {
server: Server,
_upstream: Server,
registry: PostgresRegistry,
key: String,
counters: Arc<CounterSet>,
calls: Arc<AtomicUsize>,
}
impl Endpoint {
async fn started() -> Self {
// First caller owns the process-wide fixture and its atexit cleanup, including RED panics.
let _ =
crank_test_support::postgres_schema_url_for_task("rmcp_runtime_owner", "t_c559abb1")
.await;
let registry = test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream = serve(
Router::new()
.route("/crm/leads", post(result_serialization::upstream))
.with_state(calls.clone()),
)
.await;
let operation = test_operation(&upstream.1);
registry
.create_operation(&workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &test_time(),
published_by: Some("alice"),
})
.await
.unwrap();
publish_agent(&registry, &operation).await;
let key = create_api_key(&registry).await;
let counters = Arc::new(CounterSet::default());
let app = crate::build_app(
registry.clone(),
Duration::ZERO,
Some("https://crank.example.com".into()),
SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(),
crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build(),
RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()),
Arc::new(InMemoryCoordinationStateStore::default()),
Arc::new(crate::session::InMemorySessionStore::default()),
Arc::new(crate::auth::CommunityMachineCredentialVerifier),
);
let app = observe_requests(app, counters.clone(), None).layer(axum::middleware::from_fn({
let counters = counters.clone();
move |request: axum::extract::Request, next: axum::middleware::Next| {
let counters = counters.clone();
async move {
let response = next.run(request).await;
result_serialization::observe_body(response, counters)
}
}
}));
let server = serve(app).await;
Self {
server,
_upstream: upstream,
registry,
key,
counters,
calls,
}
}
async fn post(&self, revision: &str, session: Option<&str>, body: &Value) -> reqwest::Response {
self.post_accept(revision, session, body, "application/json")
.await
}
async fn post_accept(
&self,
revision: &str,
session: Option<&str>,
body: &Value,
accept: &str,
) -> reqwest::Response {
let mut request = reqwest::Client::new()
.post(format!(
"{}/v1/{WORKSPACE_SLUG}/{AGENT_SLUG}",
self.server.1
))
.bearer_auth(&self.key)
// Also guards the legacy JSON-only contract; no SSE parsing fallback here.
.header("Accept", accept)
.header("x-request-id", "sdk-result-wire")
.header(
"traceparent",
"00-11111111111111111111111111111111-2222222222222222-01",
)
.header("MCP-Protocol-Version", revision);
if let Some(session) = session {
request = request.header("MCP-Session-Id", session);
}
if revision == MODERN_REVISION {
request = request.header("Mcp-Method", body["method"].as_str().unwrap());
if let Some(name) = body["params"]["name"].as_str() {
request = request.header("Mcp-Name", name);
}
}
request.json(body).send().await.unwrap()
}
async fn session(&self, revision: &str) -> Option<String> {
if revision == MODERN_REVISION {
return None;
}
let response = self.post(revision, None, &legacy_initialize(1)).await;
assert_eq!(response.status(), 200);
let session = response.headers()["MCP-Session-Id"]
.to_str()
.unwrap()
.to_owned();
let body = json_body(response).await;
assert_eq!(body["result"]["protocolVersion"], revision);
assert!(body["result"]["capabilities"]["tools"].is_object());
let response = self
.post(revision, Some(&session), &initialized_notification())
.await;
assert_eq!(response.status(), 202);
Some(session)
}
async fn list(&self, revision: &str, session: Option<&str>, expected_name: &str) -> usize {
let before = self.counters.sdk_tools_dispatch();
let mut body = json!({"jsonrpc":"2.0", "id":10, "method":"tools/list", "params":{}});
if revision == MODERN_REVISION {
body["params"]["_meta"] = modern_meta();
}
let calls = self.calls.load(Ordering::SeqCst);
let response = self.post(revision, session, &body).await;
assert_eq!(response.status(), 200);
assert!(
response.headers()["content-type"]
.to_str()
.unwrap()
.starts_with("application/json")
);
let body = json_body(response).await;
assert_eq!(body["id"], 10);
assert_eq!(body["result"]["tools"].as_array().unwrap().len(), 1);
assert_eq!(body["result"]["tools"][0]["name"], expected_name);
assert!(body["error"].is_null(), "{body}");
assert_eq!(
self.calls.load(Ordering::SeqCst),
calls,
"listing must not execute upstream"
);
self.counters.sdk_tools_dispatch() - before
}
async fn call(&self, revision: &str, session: Option<&str>, name: &str, email: &str) -> usize {
let before = self.counters.sdk_tools_dispatch();
let arrivals = self.calls.load(Ordering::SeqCst);
let mut body = if revision == MODERN_REVISION {
modern_tool_call(11, email)
} else {
legacy_tool_call(11, email)
};
body["params"]["name"] = json!(name);
let response = self.post(revision, session, &body).await;
assert_eq!(response.status(), 200);
assert!(
response.headers()["content-type"]
.to_str()
.unwrap()
.starts_with("application/json")
);
let body = json_body(response).await;
assert_eq!(body["id"], 11);
assert!(body["error"].is_null(), "{body}");
assert_ne!(body["result"]["isError"], true, "{body}");
assert_scoped_output(revision, &body, email);
if revision == MODERN_REVISION {
assert_eq!(body["result"]["resultType"], "complete");
}
assert_eq!(self.calls.load(Ordering::SeqCst) - arrivals, 1);
self.counters.sdk_tools_dispatch() - before
}
async fn publish_renamed_tool(&self) {
let now = time::OffsetDateTime::now_utc();
let version = AgentVersion {
agent_id: agent_id(),
version: 2,
status: AgentStatus::Draft,
instructions: json!({}),
tool_selection_policy: ToolSelectionPolicy::default(),
created_at: now,
};
let bindings = [AgentOperationBinding {
operation_id: crank_core::OperationId::new("op_create_lead"),
operation_version: 1,
tool_name: "new_publication_tool".into(),
tool_title: "New Tool".into(),
tool_description_override: None,
enabled: true,
}];
self.registry
.create_agent_draft_version(CreateAgentDraftVersionRequest {
workspace_id: &workspace_id(),
agent_id: &agent_id(),
version: &version,
bindings: &bindings,
updated_at: &now,
})
.await
.unwrap();
self.registry
.publish_agent(PublishAgentRequest {
workspace_id: &workspace_id(),
agent_id: &agent_id(),
version: 2,
published_at: &now,
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
}
}
async fn endpoint_control(revision: &str) -> [usize; 2] {
let h = Endpoint::started().await;
let session = h.session(revision).await;
let session = session.as_deref();
let list_dispatches = h.list(revision, session, TOOL_NAME).await;
let call_dispatches = h
.call(
revision,
session,
TOOL_NAME,
&format!("{revision}@example.test"),
)
.await;
// A fresh publication must be visible without reinitialize or a new endpoint.
h.publish_renamed_tool().await;
h.list(revision, session, "new_publication_tool").await;
h.call(
revision,
session,
"new_publication_tool",
"new-publication@example.test",
)
.await;
h.registry
.revoke_platform_api_key(
&workspace_id(),
&PlatformApiKeyId::new("pk_dispatch_observer"),
&time::OffsetDateTime::now_utc(),
)
.await
.unwrap();
let before_sdk = h.counters.sdk_tools_dispatch();
let arrivals = h.calls.load(Ordering::SeqCst);
let mut denied = if revision == MODERN_REVISION {
modern_tool_call(12, "denied@example.test")
} else {
legacy_tool_call(12, "denied@example.test")
};
denied["params"]["name"] = json!("new_publication_tool");
assert_eq!(h.post(revision, session, &denied).await.status(), 401);
assert_eq!(
h.calls.load(Ordering::SeqCst),
arrivals,
"revoked credential must not execute upstream"
);
assert_eq!(
h.counters.sdk_tools_dispatch(),
before_sdk,
"revocation must reject before SDK dispatch"
);
eprintln!(
"existing GREEN {revision}: JSON list/call, scoped output, upstream=2, publication refresh, revoked=401/no outbound; SDK list/call deltas={:?}",
[list_dispatches, call_dispatches]
);
[list_dispatches, call_dispatches]
}
#[tokio::test]
async fn rmcp_runtime_existing_wire_modern() {
endpoint_control(MODERN_REVISION).await;
}
#[tokio::test]
async fn rmcp_runtime_existing_wire_legacy() {
endpoint_control(LEGACY_REVISION).await;
}
#[tokio::test]
async fn rmcp_runtime_sdk_dispatch_modern() {
let counts = endpoint_control(MODERN_REVISION).await;
assert_eq!(
counts,
[1, 1],
"real endpoint tools/list and tools/call must each enter SDK typed dispatch once (structural seam oracle)"
);
}
#[tokio::test]
async fn rmcp_runtime_sdk_dispatch_legacy() {
let counts = endpoint_control(LEGACY_REVISION).await;
assert_eq!(
counts,
[1, 1],
"real endpoint tools/list and tools/call must each enter SDK typed dispatch once (structural seam oracle)"
);
}
#[tokio::test]
async fn rmcp_runtime_observer_is_request_scoped() {
let counters = Arc::new(CounterSet::default());
sdk_tools_dispatch_enter();
assert_eq!(counters.sdk_tools_dispatch(), 0);
observe(counters.clone(), async {
sdk_tools_dispatch_enter();
})
.await;
assert_eq!(counters.sdk_tools_dispatch(), 1);
let other = Arc::new(CounterSet::default());
observe(other.clone(), async {
sdk_tools_dispatch_enter();
})
.await;
assert_eq!(other.sdk_tools_dispatch(), 1);
assert_eq!(counters.sdk_tools_dispatch(), 1);
}
@@ -0,0 +1,178 @@
//! Behavioural ATDD for the rmcp 3.7.0 Tools adapter over the existing typed Tools service.
use std::sync::Arc;
use std::time::Duration;
use crank_core::{
AdmissionSchedulerConfig, CoordinationStateStore, CorrelationContext, ExecutionLocale,
MachineAccessMode, OperationSecurityLevel, PlatformApiKeyScope,
};
use crank_registry::{AdmissionLimits, MigrationAuthority, PostgresRegistry, PostgresTaskStore};
use crank_runtime::{
InMemoryCoordinationStateStore, RequestRateLimitConfig, RequestRateLimiter, SecretCrypto,
TaskService,
};
use rmcp::model::{CallToolRequestParams, PaginatedRequestParams, RequestId};
use rmcp::service::{RequestContext, serve_directly};
use rmcp::{RoleServer, ServerHandler};
use super::{
AppState,
rmcp_tools::{RmcpRequestIdentity, RmcpTools},
};
use crate::app::TaskApplication;
use crate::auth::{CommunityMachineCredentialVerifier, VerifiedMachineCredential};
use crate::catalog::PublishedToolCatalog;
use crate::session::{ActiveSessionMetrics, InMemorySessionStore};
use crate::transport::AllowedOrigins;
pub(super) async fn test_state() -> Arc<AppState> {
let database_url =
crank_test_support::postgres_schema_url_for_task("rmcp_tools_tests", "t_fdcc7df8").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
let coordination: Arc<dyn CoordinationStateStore> =
Arc::new(InMemoryCoordinationStateStore::default());
let sessions = Arc::new(InMemorySessionStore::default());
let session_metrics = ActiveSessionMetrics::start(sessions.clone());
let catalog = PublishedToolCatalog::new(registry.clone(), Duration::ZERO, coordination.clone());
let tasks_store = PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits::default(),
AdmissionSchedulerConfig::default(),
)
.expect("default admission limits and scheduler weights are valid");
let tasks = Arc::new(TaskApplication::new(
registry.clone(),
Duration::from_secs(60),
coordination.clone(),
TaskService::new(Arc::new(tasks_store)),
));
Arc::new(AppState {
registry,
catalog,
runtime: crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build(),
api_rate_limiter: RequestRateLimiter::new(RequestRateLimitConfig {
requests_per_second: 10_000,
burst: 10_000,
}),
artifact_store: None,
secret_crypto: SecretCrypto::new("01234567890123456789012345678901")
.expect("test master key"),
sessions,
session_metrics,
session_slots: Arc::new(tokio::sync::Semaphore::new(16)),
credential_verifier: Arc::new(CommunityMachineCredentialVerifier),
allowed_origins: AllowedOrigins::new(None),
logging_levels: Arc::new(crate::protocol::LogLevelStore::new()),
signal_streams: Arc::new(crate::protocol::StreamRegistry::new()),
in_flight: Arc::new(super::lifecycle::InFlightCalls::new(
super::lifecycle::MAX_IN_FLIGHT_CALLS,
)),
tasks,
})
}
fn verified_credential() -> VerifiedMachineCredential {
VerifiedMachineCredential {
machine_access_mode: MachineAccessMode::StaticAgentKey,
max_security_level: OperationSecurityLevel::Standard,
scopes: vec![PlatformApiKeyScope::Write],
platform_api_key_id: None,
}
}
fn rmcp_tools(state: Arc<AppState>) -> RmcpTools {
RmcpTools {
state,
active: Arc::default(),
}
}
async fn request_context(adapter: &RmcpTools) -> RequestContext<RoleServer> {
let (client_io, server_io) = tokio::io::duplex(1024);
let (server_reader, server_writer) = tokio::io::split(server_io);
let running = serve_directly::<RoleServer, _, _, _, _>(
adapter.clone(),
(server_reader, server_writer),
None,
);
// keep client side alive for transport lifetime
let _ = client_io;
let mut context = RequestContext::new(RequestId::Number(1), running.peer().clone());
let (mut parts, _) = axum::http::Request::new(()).into_parts();
parts.extensions.insert(RmcpRequestIdentity {
workspace_slug: "missing-workspace".into(),
agent_slug: "missing-agent".into(),
credential: verified_credential(),
correlation: CorrelationContext::generate(),
continuation: crate::protocol::ContinuationSupport::from_supported(false),
locale: ExecutionLocale::En,
session_id: None,
progress_meta: None,
sse_acceptable: false,
});
context.extensions.insert(parts);
context
}
#[tokio::test]
async fn rmcp_adapter_rejects_missing_request_identity() {
let adapter = rmcp_tools(test_state().await);
let mut context = request_context(&adapter).await;
context.extensions.remove::<axum::http::request::Parts>();
let result = <RmcpTools as ServerHandler>::list_tools(&adapter, None, context).await;
assert!(
result
.unwrap_err()
.message
.contains("missing authenticated request context")
);
}
#[tokio::test]
async fn rmcp_adapter_list_tools_reports_catalog_unavailable_for_unknown_agent() {
let state = test_state().await;
let adapter = rmcp_tools(state);
let ctx = request_context(&adapter).await;
match <RmcpTools as ServerHandler>::list_tools(
&adapter,
Some(PaginatedRequestParams::default()),
ctx,
)
.await
{
Err(err) => assert!(
err.message.contains("crate catalog unavailable"),
"unexpected error: {}",
err.message
),
Ok(_) => panic!("expected error outcome for unknown agent"),
}
}
#[tokio::test]
async fn rmcp_adapter_call_tool_reports_catalog_unavailable_for_unknown_agent() {
let state = test_state().await;
let adapter = rmcp_tools(state);
let ctx = request_context(&adapter).await;
let request = CallToolRequestParams::new("missing_tool");
match <RmcpTools as ServerHandler>::call_tool(&adapter, request, ctx).await {
Err(err) => assert!(
err.message.contains("crate catalog unavailable"),
"unexpected error: {}",
err.message
),
Ok(_) => panic!("expected error outcome for unknown agent"),
}
}
+20 -9
View File
@@ -75,11 +75,16 @@ pub mod domain {
pub use crate::protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
pub use crate::secret::{Secret, SecretKind, SecretStatus, SecretVersion};
pub use crate::task::{
TASK_APPROVAL_REFERENCE_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES, TASK_IDENTITY_MAX_BYTES,
AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase,
AttemptResolution, BeginAttempt, TASK_APPROVAL_REFERENCE_MAX_BYTES,
TASK_CANCEL_REASON_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES,
TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES,
TASK_IDENTITY_MAX_BYTES, TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS,
TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest,
TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskSupportIds,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome,
TaskCancelRequest, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskOwnerKind,
TaskPins, TaskPurgeReport, TaskRecord, TaskRetryPolicy, TaskState, TaskSupportIds,
};
pub use crate::tool_catalog::{ToolCatalogAnalysis, ToolCatalogBudget};
pub use crate::tool_quality::{
@@ -202,11 +207,17 @@ pub use product_event::{
pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion};
pub use task::{
TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT,
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
TaskState, TaskStore, TaskSupportIds, redact_task_envelope, sensitive_task_key,
validate_task_envelope,
AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase,
AttemptResolution, BeginAttempt, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES,
TASK_INPUT_PROMPT_CODE_MAX_BYTES, TASK_INPUT_SCHEMA_FORMAT_VERSION,
TASK_INPUT_WAIT_ID_MAX_BYTES, TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS,
TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest,
TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome, TaskCancelRequest, TaskEnvelope,
TaskError, TaskFinishRequest, TaskId, TaskInputRequest, TaskInputStore, TaskInputUpdateOutcome,
TaskInputUpdateRequest, TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
TaskRetryPolicy, TaskState, TaskStore, TaskSupportIds, contains_sensitive_task_keys,
redact_task_envelope, sensitive_task_key, validate_task_envelope,
};
pub use tool_catalog::{
RECOMMENDED_TOOL_CATALOG_CONTEXT_TOKENS, ToolCatalogAnalysis, ToolCatalogAnalysisError,
+285
View File
@@ -0,0 +1,285 @@
//! Durable attempt, retry-snapshot and cancellation vocabulary of a Task (Story 4.4).
use serde::{Deserialize, Serialize};
use super::{TASK_IDENTITY_MAX_BYTES, TaskAccess, TaskEnvelope, TaskError, TaskId, TaskRecord};
/// Upper bound of the retry budget a Task may pin at acceptance (Story 4.4).
pub const TASK_MAX_ATTEMPTS: u32 = 5;
/// Upper bound of the pinned backoff step between two durable attempts (Story 4.4).
pub const TASK_MAX_BACKOFF_STEP_MS: i64 = 1_000;
/// Bound of a stored cancellation reason.
pub const TASK_CANCEL_REASON_MAX_BYTES: usize = 160;
/// Immutable retry snapshot of one accepted Task.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct TaskRetryPolicy {
/// Total durable outbound attempts (1 = a single attempt, never retried).
pub max_attempts: u32,
/// Pinned base backoff between two attempts; the delay grows by this step per attempt.
pub backoff_step_ms: i64,
}
impl TaskRetryPolicy {
/// A Task that may be dispatched exactly once (legacy Tasks and unverified writes).
pub const fn single_attempt() -> Self {
Self {
max_attempts: 1,
backoff_step_ms: 0,
}
}
pub fn new(max_attempts: u32, backoff_step_ms: i64) -> Self {
Self {
max_attempts,
backoff_step_ms,
}
}
pub fn validate(&self) -> Result<(), TaskError> {
if !(1..=TASK_MAX_ATTEMPTS).contains(&self.max_attempts) {
return Err(TaskError::invalid("task retry budget is invalid"));
}
if !(0..=TASK_MAX_BACKOFF_STEP_MS).contains(&self.backoff_step_ms) {
return Err(TaskError::invalid("task retry backoff is invalid"));
}
Ok(())
}
/// Delay before attempt number `next_ordinal` (2..=max_attempts), capped so a pinned snapshot can
/// never stretch the accepted execution deadline on its own.
pub fn backoff_before(&self, next_ordinal: u32) -> i64 {
let steps = i64::from(next_ordinal.saturating_sub(1)).max(1);
self.backoff_step_ms
.saturating_mul(steps)
.min(self.backoff_step_ms.saturating_mul(4))
}
}
/// Certainty of one durable attempt outcome.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AttemptCertainty {
/// The outcome of the outbound operation is proven (success or a definite failure).
Known,
/// The operation may have been applied upstream; the effect cannot be proven locally.
Unknown,
}
impl AttemptCertainty {
pub const fn as_str(self) -> &'static str {
match self {
Self::Known => "known",
Self::Unknown => "unknown",
}
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
match value {
"known" => Ok(Self::Known),
"unknown" => Ok(Self::Unknown),
_ => Err(TaskError::invalid("unknown attempt certainty")),
}
}
}
/// Phase of one durable attempt at the moment it was resolved.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AttemptPhase {
/// The durable checkpoint exists but no outbound send was observed.
BeforeDispatch,
/// The request was handed to the transport; its effect may exist upstream.
Dispatched,
}
impl AttemptPhase {
pub const fn as_str(self) -> &'static str {
match self {
Self::BeforeDispatch => "before_dispatch",
Self::Dispatched => "dispatched",
}
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
match value {
"before_dispatch" => Ok(Self::BeforeDispatch),
"dispatched" => Ok(Self::Dispatched),
_ => Err(TaskError::invalid("unknown attempt phase")),
}
}
}
/// Closed outcome vocabulary of one resolved attempt.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AttemptOutcomeClass {
Completed,
Failed,
/// A bounded further attempt was scheduled instead of a terminal outcome.
RetryScheduled,
OutcomeUnknown,
}
impl AttemptOutcomeClass {
pub const fn as_str(self) -> &'static str {
match self {
Self::Completed => "completed",
Self::Failed => "failed",
Self::RetryScheduled => "retry_scheduled",
Self::OutcomeUnknown => "outcome_unknown",
}
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
match value {
"completed" => Ok(Self::Completed),
"failed" => Ok(Self::Failed),
"retry_scheduled" => Ok(Self::RetryScheduled),
"outcome_unknown" => Ok(Self::OutcomeUnknown),
_ => Err(TaskError::invalid("unknown attempt outcome")),
}
}
}
/// One durable outbound attempt. The row is created by the dispatch checkpoint before any network
/// use and becomes immutable once resolved, so a stale worker can never append to the history.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskAttempt {
pub attempt_id: String,
pub task_id: TaskId,
pub ordinal: u32,
pub owner: String,
pub fence: u64,
/// Canonical fingerprint of the sealed executable input this attempt was admitted for.
pub input_fingerprint: String,
/// Digest of the stable upstream idempotency key (never the raw key or headers).
pub upstream_key_digest: Option<String>,
/// Whether a retry of this attempt is safe (read or server-verified idempotent write).
pub retry_allowed: bool,
pub checkpoint_at_unix_ms: i64,
pub resolved_at_unix_ms: Option<i64>,
pub outcome: Option<AttemptOutcomeClass>,
pub certainty: Option<AttemptCertainty>,
pub phase: Option<AttemptPhase>,
}
/// Committed dispatch intent of one durable attempt (Story 4.4). The store owns the transaction: the
/// attempt row, the incremented budget and the new Task version commit together before the network is
/// touched, so an uncommitted checkpoint can never produce an outbound call.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct BeginAttempt {
pub task_id: TaskId,
pub owner: String,
pub fence: u64,
/// Version the worker read from the lease; a concurrent change rejects the checkpoint.
pub expected_version: u32,
pub attempt_id: String,
pub input_fingerprint: String,
pub upstream_key_digest: Option<String>,
/// The runtime's decision that a retry of this attempt is safe; the store still enforces the
/// pinned budget and the absolute deadline.
pub retry_allowed: bool,
}
impl BeginAttempt {
pub fn validate(&self) -> Result<(), TaskError> {
if self.owner.is_empty() || self.owner.len() > TASK_IDENTITY_MAX_BYTES {
return Err(TaskError::invalid("task attempt owner is invalid"));
}
if self.attempt_id.is_empty() || self.attempt_id.len() > TASK_IDENTITY_MAX_BYTES {
return Err(TaskError::invalid("task attempt identity is invalid"));
}
if self.input_fingerprint.len() != 64 {
return Err(TaskError::invalid("task attempt fingerprint is invalid"));
}
if let Some(digest) = self.upstream_key_digest.as_deref()
&& (digest.is_empty() || digest.len() > TASK_IDENTITY_MAX_BYTES)
{
return Err(TaskError::invalid(
"task attempt upstream key digest is invalid",
));
}
Ok(())
}
}
/// Leased, committed view of one in-flight attempt and the Task version it was admitted under.
#[derive(Clone, Debug, PartialEq)]
pub struct AttemptLease {
pub attempt: TaskAttempt,
pub task: TaskRecord,
}
/// Typed outcome of one attempt, handed to the store for a single atomic resolution.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum AttemptOutcome {
/// Proven success; the Task becomes `completed` with this result.
Completed(TaskEnvelope),
/// Proven failure. `retryable` only schedules another attempt when the store's pinned budget,
/// the attempt's safe-retry decision and the absolute deadline all still allow it; otherwise the
/// Task fails with this error.
Failed {
error: TaskEnvelope,
retryable: bool,
phase: AttemptPhase,
},
/// The outbound effect cannot be proven. `repeatable` only schedules another bounded attempt when
/// repeating it is safe and the pinned budget/deadline still allow it; otherwise the Task becomes
/// the non-terminal `outcome_unknown`, never a plain failure.
Uncertain {
error: TaskEnvelope,
repeatable: bool,
phase: AttemptPhase,
},
}
/// Single-winner result of resolving one attempt.
#[derive(Clone, Debug, PartialEq)]
pub enum AttemptResolution {
/// A terminal transition committed (completed or failed).
Finished(TaskRecord),
/// The attempt failed in a known way and another bounded attempt was armed.
RetryAt {
task: TaskRecord,
next_attempt_at_unix_ms: i64,
},
/// The attempt resolved to the non-terminal uncertain state.
Unknown(TaskRecord),
}
/// Owner-side cancellation request. `expected_version` is optional so a repeated cancel of the same
/// accepted Task stays idempotent while a stale version is reported as a conflict.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskCancelRequest {
pub access: TaskAccess,
pub task_id: TaskId,
pub expected_version: Option<u32>,
pub reason: String,
}
impl TaskCancelRequest {
pub fn validate(&self) -> Result<(), TaskError> {
self.access.validate()?;
if self.reason.is_empty()
|| self.reason.len() > TASK_CANCEL_REASON_MAX_BYTES
|| self.reason.bytes().any(|byte| byte.is_ascii_control())
{
return Err(TaskError::invalid("task cancellation reason is invalid"));
}
Ok(())
}
}
/// Typed result of an owner-scoped cancellation.
#[derive(Clone, Debug, PartialEq)]
pub enum TaskCancelOutcome {
/// Cancelled before the dispatch checkpoint: no outbound call can have happened.
Cancelled(TaskRecord),
/// The Task already had a checkpointed attempt: non-terminal uncertain, never a plain cancel.
OutcomeUnknown(TaskRecord),
/// The Task was already in a terminal state; its accepted result is unchanged.
AlreadyTerminal(TaskRecord),
/// The caller's expected version is stale; `current` is the authorized current state.
Conflict(TaskRecord),
}
+295
View File
@@ -0,0 +1,295 @@
//! Continuation of an accepted Task through a validated additional input (Story 4.5, FR-25).
//!
//! A Task that cannot run because the pinned operation still lacks a required input item is
//! suspended as `input_required` before any outbound dispatch. The durable request keeps the exact
//! immutable schema snapshot, its JSON projection, safe prompt metadata, the absolute TTL and the
//! update fingerprint contract. A valid update moves the Task back to `working` exactly once; an
//! identical replay returns the committed outcome and a conflicting or stale reply is typed.
//!
//! Deliberately absent: raw input in the request/receipt/audit read models, a second cipher store and
//! any public transport. The regular executable-input ciphertext carries the continuation binding.
use serde_json::Value;
use crate::task::{TaskAccess, TaskError, TaskId};
/// Version of the persisted input-continuation contract. A stored snapshot always declares the
/// version it was written with, so a later binary never reinterprets it silently.
pub const TASK_INPUT_SCHEMA_FORMAT_VERSION: u32 = 1;
/// Bound of a wait identity before it is persisted.
pub const TASK_INPUT_WAIT_ID_MAX_BYTES: usize = 64;
/// Bound of the safe prompt code (never the prompt text or an input value).
pub const TASK_INPUT_PROMPT_CODE_MAX_BYTES: usize = 64;
/// Bound of the missing-field path metadata.
pub const TASK_INPUT_MAX_MISSING_FIELDS: usize = 32;
/// Bound of one missing-field path.
pub const TASK_INPUT_FIELD_PATH_MAX_BYTES: usize = 160;
/// Durable request of additional input for one suspended Task.
///
/// `schema_snapshot` is the native serialized schema subset the server validates against; the JSON
/// projection is presentation metadata only and never the validation authority. `expected_version`
/// is the Task version a caller must present with the answer.
#[derive(Clone, Debug, PartialEq)]
pub struct TaskInputRequest {
pub wait_id: String,
pub expected_version: u32,
pub schema_revision: u32,
pub schema_format_version: u32,
pub schema_snapshot: Value,
pub projected_schema: Value,
pub prompt_code: String,
pub missing_fields: Vec<String>,
pub expires_at_unix_ms: i64,
pub fingerprint_format_version: u32,
}
impl TaskInputRequest {
pub fn validate(&self) -> Result<(), TaskError> {
if self.wait_id.is_empty()
|| self.wait_id.len() > TASK_INPUT_WAIT_ID_MAX_BYTES
|| self.wait_id.bytes().any(|byte| byte.is_ascii_control())
{
return Err(TaskError::invalid("task input wait identity is invalid"));
}
if self.expected_version == 0 || self.schema_revision == 0 {
return Err(TaskError::invalid(
"task input request generation is invalid",
));
}
if self.schema_format_version < TASK_INPUT_SCHEMA_FORMAT_VERSION
|| self.fingerprint_format_version < TASK_INPUT_SCHEMA_FORMAT_VERSION
{
return Err(TaskError::invalid(
"task input request format version is invalid",
));
}
if self.schema_snapshot.as_object().is_none() || self.projected_schema.as_object().is_none()
{
return Err(TaskError::invalid("task input schema snapshot is invalid"));
}
if self.prompt_code.is_empty() || self.prompt_code.len() > TASK_INPUT_PROMPT_CODE_MAX_BYTES
{
return Err(TaskError::invalid("task input prompt code is invalid"));
}
if self.missing_fields.len() > TASK_INPUT_MAX_MISSING_FIELDS
|| self
.missing_fields
.iter()
.any(|field| field.is_empty() || field.len() > TASK_INPUT_FIELD_PATH_MAX_BYTES)
{
return Err(TaskError::invalid(
"task input missing-field metadata is invalid",
));
}
if self.expires_at_unix_ms <= 0 {
return Err(TaskError::invalid("task input TTL is invalid"));
}
Ok(())
}
}
/// Authoritative write of one validated answer. The transaction owns schema-generation, TTL,
/// fingerprint and CAS checks; this request only carries the committed values.
#[derive(Clone, Debug, PartialEq)]
pub struct TaskInputUpdateRequest {
pub access: TaskAccess,
pub task_id: TaskId,
pub wait_id: String,
pub expected_version: u32,
/// Digest of the scoped update key; the caller key is never persisted.
pub update_key_digest: String,
/// Canonical digest of the validated full executable input.
pub canonical_fingerprint: String,
/// Effective execution fingerprint of the resumed payload (never the acceptance fingerprint).
pub effective_input_fingerprint: String,
pub executable_input_ciphertext: String,
pub executable_input_key_version: String,
pub executable_input_master_key_epoch: i64,
}
impl TaskInputUpdateRequest {
pub fn validate(&self) -> Result<(), TaskError> {
self.access.validate()?;
if self.wait_id.is_empty() || self.wait_id.len() > TASK_INPUT_WAIT_ID_MAX_BYTES {
return Err(TaskError::invalid("task input wait identity is invalid"));
}
if self.expected_version == 0 {
return Err(TaskError::invalid("task input version is invalid"));
}
if !valid_digest(&self.update_key_digest)
|| !valid_digest(&self.canonical_fingerprint)
|| !valid_digest(&self.effective_input_fingerprint)
{
return Err(TaskError::invalid("task input digest is invalid"));
}
if self.executable_input_ciphertext.is_empty()
|| self.executable_input_ciphertext.len()
> crate::task::TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES
{
return Err(TaskError::invalid(
"task continuation ciphertext is invalid",
));
}
if self.executable_input_key_version.is_empty()
|| self.executable_input_key_version.len() > 64
|| self.executable_input_master_key_epoch < 1
{
return Err(TaskError::invalid(
"task continuation ciphertext is invalid",
));
}
Ok(())
}
}
/// Committed outcome of one input update. An identical replay returns the same stored outcome with
/// `replayed = true` instead of repeating the CAS, audit or dispatch.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskInputUpdateOutcome {
pub task_id: TaskId,
pub wait_id: String,
pub state: crate::task::TaskState,
pub version: u32,
pub replayed: bool,
}
fn valid_digest(value: &str) -> bool {
value.len() == 64
&& value
.bytes()
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
}
/// Durable input-continuation operations of the Task store.
///
/// Split out of the general [`crate::task::TaskStore`] port so the lifecycle store stays readable and
/// the continuation contract (suspend, answer, receipt, expiry) is implemented and audited as one
/// cohesive surface. [`crate::task::TaskStore`] extends this trait, so an owned `Arc<dyn TaskStore>`
/// exposes both halves through one object.
#[async_trait::async_trait]
pub trait TaskInputStore: Send + Sync {
/// Suspends a leased Task that still lacks a required input of its pinned operation. The
/// transaction persists the immutable request and its audit row, moves `working` to
/// `input_required` exactly once, releases the worker lease and keeps the existing admission
/// permit. No outbound dispatch may have happened in this generation.
async fn suspend_for_input(
&self,
lease: &crate::task::TaskLease,
request: TaskInputRequest,
) -> Result<crate::task::TaskRecord, TaskError>;
/// Owner-scoped read of the durable input request of one Task.
async fn input_request(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<TaskInputRequest, TaskError>;
/// Owner-scoped read of the executable-input binding that a committed continuation may extend.
/// It is not lease-authorized: it only serves the update transaction and never dispatch.
async fn read_executable_input_for_update(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<(String, String, i64), TaskError>;
/// Applies exactly one validated answer under a row lock: authority recheck, receipt lookup,
/// generation/TTL check, atomic ciphertext/effective-fingerprint/receipt/`working` transition and
/// audit. An identical replay returns the stored outcome without a second transition.
async fn apply_input(
&self,
request: TaskInputUpdateRequest,
) -> Result<TaskInputUpdateOutcome, TaskError>;
/// Bounded sweep of `input_required` Tasks whose TTL elapsed: `failed` with an audit reason and
/// the release of the admission permit, without any dispatch.
async fn expire_input_waits(&self, batch: u32) -> Result<u64, TaskError>;
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn request() -> TaskInputRequest {
TaskInputRequest {
wait_id: "wait-1".to_owned(),
expected_version: 2,
schema_revision: 1,
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
schema_snapshot: json!({"type": "object", "fields": {}}),
projected_schema: json!({"type": "object", "properties": {}}),
prompt_code: "required_input_missing".to_owned(),
missing_fields: vec!["$.email".to_owned()],
expires_at_unix_ms: 1_900_000_000_000,
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
}
}
fn update() -> TaskInputUpdateRequest {
TaskInputUpdateRequest {
access: crate::task::TaskAccess {
workspace_id: "ws_default".to_owned(),
owner_kind: crate::task::TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
},
task_id: crate::task::TaskId::generate(),
wait_id: "wait-1".to_owned(),
expected_version: 2,
update_key_digest: "a".repeat(64),
canonical_fingerprint: "b".repeat(64),
effective_input_fingerprint: "c".repeat(64),
executable_input_ciphertext: "sealed".to_owned(),
executable_input_key_version: "key-v1".to_owned(),
executable_input_master_key_epoch: 1,
}
}
#[test]
fn input_request_holds_generation_schema_ttl_and_bounds() {
assert!(request().validate().is_ok());
let mut no_wait = request();
no_wait.wait_id.clear();
assert!(no_wait.validate().is_err());
let mut no_generation = request();
no_generation.expected_version = 0;
assert!(no_generation.validate().is_err());
let mut bad_schema = request();
bad_schema.schema_snapshot = json!("not-an-object");
assert!(bad_schema.validate().is_err());
let mut no_ttl = request();
no_ttl.expires_at_unix_ms = 0;
assert!(no_ttl.validate().is_err());
let mut too_many_fields = request();
too_many_fields.missing_fields = vec!["$.x".to_owned(); TASK_INPUT_MAX_MISSING_FIELDS + 1];
assert!(too_many_fields.validate().is_err());
}
#[test]
fn update_request_rejects_invalid_digests_and_ciphertext_bounds() {
assert!(update().validate().is_ok());
let mut short_digest = update();
short_digest.canonical_fingerprint = "abc".to_owned();
assert!(short_digest.validate().is_err());
let mut upper_digest = update();
upper_digest.update_key_digest = "A".repeat(64);
assert!(upper_digest.validate().is_err());
let mut no_ciphertext = update();
no_ciphertext.executable_input_ciphertext.clear();
assert!(no_ciphertext.validate().is_err());
let mut no_epoch = update();
no_epoch.executable_input_master_key_epoch = 0;
assert!(no_epoch.validate().is_err());
}
}
@@ -9,16 +9,25 @@
//! Deliberately absent: credentials, prepared headers, approval bearer tokens and any claim that a
//! Task was executed. A terminal state is written by a conditional transition only.
mod attempt;
mod input;
use async_trait::async_trait;
pub use attempt::*;
pub use input::*;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use uuid::Uuid;
use crate::correlation::{RequestId, TraceContext, TraceId};
use crate::ids::PlatformApiKeyId;
use crate::operation::ToolExecutionMode;
/// Byte budget of every persisted Task envelope (input, progress, result, error, schema metadata).
pub const TASK_ENVELOPE_MAX_BYTES: usize = 32_768;
/// Maximum byte budget of the decrypted executable Task input.
pub const TASK_EXECUTABLE_INPUT_MAX_BYTES: usize = TASK_ENVELOPE_MAX_BYTES;
/// Maximum byte budget of the stored encrypted executable Task input ciphertext.
pub const TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES: usize = 98_304;
/// Maximum nesting depth of a persisted Task envelope.
pub const TASK_ENVELOPE_MAX_DEPTH: usize = 8;
/// Maximum number of JSON members/nodes of a persisted Task envelope.
@@ -122,9 +131,9 @@ impl TaskState {
| Self::Cancelled
),
Self::InputRequired => matches!(next, Self::Working | Self::Failed | Self::Cancelled),
Self::OutcomeUnknown => {
matches!(next, Self::Completed | Self::Failed | Self::Cancelled)
}
// Story 4.4: an uncertain outbound outcome is never resolved by a generic transition.
// Reconciliation is Story 4.6, so no ordinary finish/cancel/retry may leave this state.
Self::OutcomeUnknown => false,
Self::Completed | Self::Failed | Self::Cancelled => false,
}
}
@@ -201,6 +210,9 @@ pub struct TaskPins {
pub retain_until_unix_ms: i64,
/// Verified approval evidence reference (identifier only, never a bearer token).
pub approval_reference: Option<String>,
/// Bounded retry budget pinned at acceptance (Story 4.4). A later publication, a restart or a
/// new binary never changes the numbers an accepted Task was admitted with.
pub retry: TaskRetryPolicy,
}
impl TaskPins {
@@ -228,6 +240,7 @@ impl TaskPins {
{
return Err(TaskError::invalid("task approval reference is invalid"));
}
self.retry.validate()?;
Ok(())
}
}
@@ -351,6 +364,16 @@ pub fn redact_task_envelope(value: &Value) -> Value {
}
}
/// Recursive check used to fail closed on sensitive fields before encryption.
pub fn contains_sensitive_task_keys(value: &Value) -> bool {
match value {
Value::Object(object) => object
.iter()
.any(|(key, child)| sensitive_task_key(key) || contains_sensitive_task_keys(child)),
Value::Array(values) => values.iter().any(contains_sensitive_task_keys),
_ => false,
}
}
/// Write request of a new durable Task. The store owns the transaction: Task, dispatch intent and
/// queue capacity commit together, and no acknowledgement exists before that commit.
#[derive(Clone, Debug, PartialEq, Eq)]
@@ -360,6 +383,10 @@ pub struct TaskAcceptRequest {
pub owner_kind: TaskOwnerKind,
pub owner_id: String,
pub authorization_scope: String,
/// Typed identity of the Platform API key that authorized the acceptance (Story 4.3). Only the
/// immutable key ID is stored, never the bearer secret; a worker revalidates it authoritatively.
/// `None` means the accepting principal has no machine key (administrator) and fail-closes.
pub platform_api_key_id: Option<PlatformApiKeyId>,
pub operation_id: String,
pub pins: TaskPins,
/// Digest of the caller key over the full owner/authorization/Tool scope.
@@ -368,6 +395,10 @@ pub struct TaskAcceptRequest {
pub input_fingerprint: String,
pub input: TaskEnvelope,
pub support: TaskSupportIds,
/// Bounded encrypted executable input, persisted alongside the redacted input projection.
pub executable_input_ciphertext: Option<String>,
pub executable_input_key_version: Option<String>,
pub executable_input_master_key_epoch: Option<i64>,
}
impl TaskAcceptRequest {
@@ -382,6 +413,13 @@ impl TaskAcceptRequest {
return Err(TaskError::invalid(format!("task {field} is invalid")));
}
}
if let Some(key_id) = self.platform_api_key_id.as_ref()
&& (key_id.as_str().is_empty() || key_id.as_str().len() > TASK_IDENTITY_MAX_BYTES)
{
return Err(TaskError::invalid(
"task platform api key reference is invalid",
));
}
if self.scoped_key.len() < 16 || self.scoped_key.len() > TASK_SCOPED_KEY_MAX_BYTES {
return Err(TaskError::invalid("task scoped key digest is invalid"));
}
@@ -394,7 +432,40 @@ impl TaskAcceptRequest {
return Err(TaskError::invalid("task input fingerprint is invalid"));
}
self.pins.validate()?;
self.support.validate()
self.support.validate()?;
let has_ciphertext = self.executable_input_ciphertext.is_some();
let has_key = self.executable_input_key_version.is_some();
let has_epoch = self.executable_input_master_key_epoch.is_some();
match (has_ciphertext, has_key, has_epoch) {
(false, false, false) => {}
(true, true, true) => {
let ciphertext = self.executable_input_ciphertext.as_deref().unwrap();
if ciphertext.is_empty()
|| ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES
{
return Err(TaskError::invalid(
"task executable input ciphertext is invalid",
));
}
let key_version = self.executable_input_key_version.as_deref().unwrap();
if key_version.is_empty() || key_version.len() > 64 {
return Err(TaskError::invalid(
"task executable input ciphertext is invalid",
));
}
if self.executable_input_master_key_epoch.unwrap_or_default() < 1 {
return Err(TaskError::invalid(
"task executable input ciphertext is invalid",
));
}
}
_ => {
return Err(TaskError::invalid(
"task executable input ciphertext is invalid",
));
}
}
Ok(())
}
}
@@ -448,9 +519,16 @@ pub struct TaskRecord {
pub owner_kind: TaskOwnerKind,
pub owner_id: String,
pub authorization_scope: String,
/// Typed immutable identity of the Platform API key that authorized the acceptance; the worker
/// revalidates this exact key against the registry before any outbound dispatch.
pub platform_api_key_id: Option<PlatformApiKeyId>,
pub operation_id: String,
/// Canonical digest of the accepted executable input (compare-and-conflict identity).
pub input_fingerprint: String,
/// Effective fingerprint of the payload the next attempt will execute. It equals the acceptance
/// fingerprint until a committed input continuation replaces the effective payload; the
/// acceptance fingerprint above is never overwritten, so an original accept replay stays replay.
pub effective_input_fingerprint: String,
pub pins: TaskPins,
pub support: TaskSupportIds,
pub input: TaskEnvelope,
@@ -458,6 +536,22 @@ pub struct TaskRecord {
pub result: Option<TaskEnvelope>,
pub error: Option<TaskEnvelope>,
pub attempts: u32,
/// Monotonic lease/fencing token. It never resets and protects Task state mutations.
pub fence: u64,
/// Random process-instance lease owner, if a worker currently owns the Task.
pub lease_owner: Option<String>,
/// Lease expiry epoch milliseconds, if the Task is claimed.
pub lease_until_unix_ms: Option<i64>,
/// Current dispatch attempt identity, committed before any outbound adapter call.
pub current_attempt_id: Option<String>,
/// Dispatch checkpoint epoch milliseconds, if an outbound attempt was committed.
pub dispatch_started_at_unix_ms: Option<i64>,
/// Earliest moment the next durable attempt may be dispatched (pinned backoff).
pub next_attempt_at_unix_ms: Option<i64>,
/// Durable cancellation request that arrived while an attempt was already checkpointed.
pub cancel_requested_at_unix_ms: Option<i64>,
/// Bounded reason of the cancellation request.
pub cancel_reason: Option<String>,
/// Monotonic lifecycle version used by the terminal CAS.
pub version: u32,
pub created_at_unix_ms: i64,
@@ -519,6 +613,8 @@ pub enum TaskError {
FingerprintConflict,
#[error("task lifecycle transition was rejected")]
StateConflict,
#[error("task input request has expired")]
InputExpired,
#[error("background task capacity is exhausted")]
Overloaded { retry_after_ms: u64 },
}
@@ -542,9 +638,18 @@ impl TaskError {
}
}
/// Leased view of a claimed Task: its durable record plus lease fencing identity.
#[derive(Clone, Debug, PartialEq)]
pub struct TaskLease {
pub record: TaskRecord,
pub owner: String,
pub fence: u64,
pub lease_until: i64,
}
/// Authoritative durable Task port.
#[async_trait]
pub trait TaskStore: Send + Sync {
pub trait TaskStore: TaskInputStore + Send + Sync {
/// Proves the durable Task state is usable (schema and operations reachable).
async fn health(&self) -> Result<(), TaskError>;
@@ -566,9 +671,78 @@ pub trait TaskStore: Send + Sync {
/// Applies exactly one conditional terminal transition and releases its intent atomically.
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError>;
/// Bounded retention: fails guaranteed-undispatched queued Tasks past their deadline and removes
/// expired terminal records. Live and non-terminal work is never deleted.
/// Reads encrypted executable input only under a valid, unexpired current lease.
async fn read_executable_input(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
) -> Result<(String, String, i64), TaskError>;
/// Bounded retention...
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError>;
/// Claims a bounded batch of queued or safely recoverable Tasks under leases.
async fn claim_batch(
&self,
owner: &str,
batch_limit: u32,
lease_ms: i64,
) -> Result<Vec<TaskLease>, TaskError>;
/// Renews a lease owned by `owner` when fence and expiry still match.
async fn renew_lease(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
lease_ms: i64,
) -> Result<TaskLease, TaskError>;
/// Persists redacted progress under a valid lease.
async fn write_progress(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
progress: TaskEnvelope,
) -> Result<TaskRecord, TaskError>;
/// Commits one durable outbound attempt before network use: attempt row, incremented budget and
/// the new Task version commit in one transaction. A returned lease is the only authority to
/// dispatch that attempt; an unknown commit means zero outbound calls.
async fn checkpoint_dispatch(&self, request: BeginAttempt) -> Result<AttemptLease, TaskError>;
/// Resolves exactly one committed attempt under its lease/fence/version. Only the current attempt
/// of the current owner may be resolved, and the attempt becomes immutable afterwards.
async fn resolve_attempt(
&self,
lease: &AttemptLease,
outcome: AttemptOutcome,
) -> Result<AttemptResolution, TaskError>;
/// Owner-scoped atomic cancellation. It serializes with claim and checkpoint on the same Task.
async fn cancel(&self, request: TaskCancelRequest) -> Result<TaskCancelOutcome, TaskError>;
/// Owner-scoped bounded history of the durable attempts of one Task.
async fn attempts(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<Vec<TaskAttempt>, TaskError>;
/// Applies a terminal transition only under a valid lease and releases intent atomically.
async fn finish_leased(
&self,
request: TaskFinishRequest,
owner: &str,
fence: u64,
) -> Result<TaskRecord, TaskError>;
/// Resolves a leased Task whose next attempt was refused before its dispatch checkpoint (elapsed
/// deadline, rejected authority). A Task that still carries a proven ambiguity is never downgraded
/// to a plain failure: it ends non-terminal `outcome_unknown` with `code`; otherwise `failed`.
async fn refuse_attempt(&self, lease: &TaskLease, code: &str) -> Result<TaskRecord, TaskError>;
}
#[cfg(test)]
@@ -583,6 +757,7 @@ mod tests {
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
platform_api_key_id: Some(PlatformApiKeyId::new("pk_1")),
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 3,
@@ -590,6 +765,7 @@ mod tests {
execution_deadline_unix_ms: 1_790_000_000_000,
retain_until_unix_ms: 1_790_000_000_000 + 1_000,
approval_reference: Some("appr_1".to_owned()),
retry: TaskRetryPolicy::new(3, 1),
},
scoped_key: "a".repeat(64),
input_fingerprint: "b".repeat(64),
@@ -599,6 +775,9 @@ mod tests {
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
.unwrap(),
),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
}
}
@@ -694,6 +873,61 @@ mod tests {
let mut bad_retention = request();
bad_retention.pins.retain_until_unix_ms = bad_retention.pins.execution_deadline_unix_ms - 1;
assert!(bad_retention.validate().is_err());
// The persisted authority reference is bounded and never empty.
let mut empty_key_reference = request();
empty_key_reference.platform_api_key_id = Some(PlatformApiKeyId::new(""));
assert!(empty_key_reference.validate().is_err());
}
#[test]
fn outcome_unknown_has_no_generic_successor() {
// Story 4.4: reconciliation of an uncertain outcome is Story 4.6. Until then no ordinary
// finish/cancel/retry may leave the non-terminal uncertain state.
for next in [
TaskState::Queued,
TaskState::Working,
TaskState::InputRequired,
TaskState::OutcomeUnknown,
TaskState::Completed,
TaskState::Failed,
TaskState::Cancelled,
] {
assert!(
!TaskState::OutcomeUnknown.allows_transition_to(next),
"outcome_unknown must not transition to {next:?}"
);
}
assert!(!TaskState::OutcomeUnknown.is_terminal());
}
#[test]
fn retry_policy_bounds_are_pinned_and_bounded() {
assert!(TaskRetryPolicy::single_attempt().validate().is_ok());
assert!(
TaskRetryPolicy::new(TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS)
.validate()
.is_ok()
);
assert!(TaskRetryPolicy::new(0, 0).validate().is_err());
assert!(
TaskRetryPolicy::new(TASK_MAX_ATTEMPTS + 1, 0)
.validate()
.is_err()
);
assert!(
TaskRetryPolicy::new(1, TASK_MAX_BACKOFF_STEP_MS + 1)
.validate()
.is_err()
);
// The pinned step grows the delay but never beyond four steps.
assert_eq!(TaskRetryPolicy::new(5, 2).backoff_before(2), 2);
assert_eq!(TaskRetryPolicy::new(5, 2).backoff_before(5), 8);
assert_eq!(TaskRetryPolicy::single_attempt().backoff_before(2), 0);
// A pin whose retry snapshot is invalid is rejected at acceptance.
let mut invalid = request();
invalid.pins.retry = TaskRetryPolicy::new(0, 0);
assert!(invalid.validate().is_err());
}
#[test]
+10
View File
@@ -6,7 +6,12 @@ mod artifact_cleanup_indexes_v13;
mod artifact_metadata_v12;
mod authority;
mod baseline_v1;
mod durable_task_input_v30;
mod durable_tasks_v25;
mod durable_tasks_v26;
mod durable_tasks_v27;
mod durable_tasks_v28;
mod durable_tasks_v29;
mod execution_admission_v24;
mod execution_outcome_v5;
mod import_plan_v14;
@@ -39,6 +44,11 @@ mod schema_guard_v22;
mod schema_guard_v23;
mod schema_guard_v24;
mod schema_guard_v25;
mod schema_guard_v26;
mod schema_guard_v27;
mod schema_guard_v28;
mod schema_guard_v29;
mod schema_guard_v30;
mod schema_guard_v7;
mod schema_guard_v8;
mod schema_guard_v9;
@@ -4,7 +4,12 @@ use super::agent_resource_bindings_v18;
use super::approval_side_effects_v10;
use super::artifact_cleanup_indexes_v13;
use super::artifact_metadata_v12;
use super::durable_task_input_v30;
use super::durable_tasks_v25;
use super::durable_tasks_v26;
use super::durable_tasks_v27;
use super::durable_tasks_v28;
use super::durable_tasks_v29;
use super::execution_admission_v24;
use super::execution_outcome_v5;
use super::import_plan_v14;
@@ -41,9 +46,10 @@ use preflight::*;
use contract::baseline_source_digest;
use contract::validate_descriptors;
const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947;
const CURRENT_VERSION: i64 = 25;
const CURRENT_VERSION: i64 = 30;
const IMPLEMENTED_VERSIONS: &[i64] = &[
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
27, 28, 29, 30,
];
const BASELINE_SOURCE_SHA256: &str =
"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675";
@@ -344,6 +350,36 @@ impl MigrationAuthority {
durable_tasks_v25::SOURCE_SHA256,
24,
),
expand_descriptor(
26,
durable_tasks_v26::NAME,
durable_tasks_v26::SOURCE_SHA256,
25,
),
expand_descriptor(
27,
durable_tasks_v27::NAME,
durable_tasks_v27::SOURCE_SHA256,
26,
),
expand_descriptor(
28,
durable_tasks_v28::NAME,
durable_tasks_v28::SOURCE_SHA256,
27,
),
expand_descriptor(
29,
durable_tasks_v29::NAME,
durable_tasks_v29::SOURCE_SHA256,
28,
),
expand_descriptor(
30,
durable_task_input_v30::NAME,
durable_task_input_v30::SOURCE_SHA256,
29,
),
]
}
pub fn validate_sequence() -> Result<(), MigrationError> {
@@ -509,6 +545,21 @@ impl MigrationAuthority {
if from < 25 {
durable_tasks_v25::apply(&mut transaction, &Self::sequence()[24]).await?;
}
if from < 26 {
durable_tasks_v26::apply(&mut transaction, &Self::sequence()[25]).await?;
}
if from < 27 {
durable_tasks_v27::apply(&mut transaction, &Self::sequence()[26]).await?;
}
if from < 28 {
durable_tasks_v28::apply(&mut transaction, &Self::sequence()[27]).await?;
}
if from < 29 {
durable_tasks_v29::apply(&mut transaction, &Self::sequence()[28]).await?;
}
if from < 30 {
durable_task_input_v30::apply(&mut transaction, &Self::sequence()[29]).await?;
}
match inspect(&mut transaction).await? {
MigrationPreflight::Current {
version: CURRENT_VERSION,
@@ -0,0 +1,45 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "durable-task-input-v30";
pub(super) const SOURCE: &str = include_str!("durable_task_input_v30.sql");
pub(super) const SOURCE_SHA256: &str =
"84a51f712fd0bc439acba2ab3849232777c29e3b181586bbb4b06fd129843a38";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_task_input_v30",
Some(30),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_task_input_v30",
Some(30),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -0,0 +1,59 @@
-- V30: continuation through validated input (Story 4.5).
alter table tasks add column if not exists effective_input_fingerprint text;
update tasks set effective_input_fingerprint = input_fingerprint where effective_input_fingerprint is null;
alter table tasks alter column effective_input_fingerprint set not null;
alter table tasks add constraint tasks_effective_input_fingerprint_check check (
length(effective_input_fingerprint) = 64 and effective_input_fingerprint ~ '^[0-9a-f]{64}$'
);
create table if not exists task_input_requests (
task_id uuid primary key references tasks (task_id) on delete cascade,
wait_id text not null,
expected_version integer not null,
schema_revision integer not null,
schema_format_version integer not null,
fingerprint_format_version integer not null,
schema_snapshot jsonb not null,
projected_schema jsonb not null,
prompt_code text not null,
missing_fields jsonb not null default '[]'::jsonb,
expires_at timestamptz not null,
created_at timestamptz not null default now(),
accepted_at timestamptz,
update_key_digest text,
canonical_fingerprint text,
effective_input_fingerprint text,
committed_state text,
committed_version integer,
constraint task_input_requests_wait_unique unique (wait_id),
constraint task_input_requests_prompt_check check (octet_length(prompt_code) between 1 and 64),
constraint task_input_requests_receipt_check check (
(accepted_at is null and update_key_digest is null and canonical_fingerprint is null
and effective_input_fingerprint is null and committed_state is null and committed_version is null)
or (accepted_at is not null and update_key_digest is not null and canonical_fingerprint is not null
and effective_input_fingerprint is not null and committed_state is not null and committed_version is not null)
)
);
-- Not unique on purpose: several waiting Tasks may share one absolute TTL value (the same pinned
-- policy deadline), and an expired/cancelled wait keeps its row. Per-Task uniqueness comes from the
-- primary key and task_input_requests_wait_unique.
create index if not exists task_input_requests_active_ttl_idx
on task_input_requests (expires_at) where accepted_at is null;
create table if not exists task_input_audit (
id bigint generated always as identity primary key,
task_id uuid not null references tasks (task_id) on delete cascade,
wait_id text not null,
event text not null,
from_state text,
to_state text,
from_version integer,
to_version integer,
actor text,
reason_code text,
created_at timestamptz not null default now(),
constraint task_input_audit_event_check check (event in ('input_required','input_accepted','input_expired')),
constraint task_input_audit_actor_check check (actor is null or octet_length(actor) between 1 and 160),
constraint task_input_audit_reason_check check (reason_code is null or octet_length(reason_code) between 1 and 64)
);
create index if not exists task_input_audit_task_idx on task_input_audit (task_id, id);
@@ -0,0 +1,45 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "durable-tasks-v26";
pub(super) const SOURCE: &str = include_str!("durable_tasks_v26.sql");
pub(super) const SOURCE_SHA256: &str =
"6dcbf9990deabb3984a333acbfc3cb490cfc886c2ec30093e3b46d9113616089";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v26",
Some(26),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v26",
Some(26),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -0,0 +1,6 @@
-- V26: PostgreSQL lease/fencing/CAS additions for durable background Tasks (Story 4.3).
alter table tasks add column if not exists fence bigint not null default 0;
alter table tasks add column if not exists lease_owner text;
alter table tasks add column if not exists lease_until timestamptz;
alter table tasks add column if not exists current_attempt_id text;
alter table tasks add column if not exists dispatch_started_at timestamptz;
@@ -0,0 +1,45 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "durable-tasks-v27";
pub(super) const SOURCE: &str = include_str!("durable_tasks_v27.sql");
pub(super) const SOURCE_SHA256: &str =
"a3c5335cbf81c244359faaee993487ae132d8b3be6b9c63fa322788021273dc8";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v27",
Some(27),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v27",
Some(27),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -0,0 +1,19 @@
-- V27: encrypted executable Task input (Story 4.3).
alter table tasks add column if not exists executable_input_ciphertext text;
alter table tasks add column if not exists executable_input_key_version text;
alter table tasks add column if not exists executable_input_master_key_epoch bigint;
alter table tasks add constraint tasks_executable_input_key_version_check check (
executable_input_ciphertext is null
or (executable_input_key_version is not null and octet_length(executable_input_key_version) between 1 and 64)
);
alter table tasks add column if not exists target_executable_input_ciphertext text;
alter table tasks add column if not exists target_executable_input_key_version text;
alter table tasks add column if not exists target_executable_input_master_key_epoch bigint;
alter table tasks add constraint tasks_executable_input_epoch_check check (
executable_input_ciphertext is null
or (executable_input_master_key_epoch is not null
and executable_input_master_key_epoch >= 1)
);
@@ -0,0 +1,45 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "durable-tasks-v28";
pub(super) const SOURCE: &str = include_str!("durable_tasks_v28.sql");
pub(super) const SOURCE_SHA256: &str =
"4dc77739c2ea1e32a4b00abd70df0f00fd82b7cf7933372f869f2c1e997a3676";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v28",
Some(28),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v28",
Some(28),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -0,0 +1,8 @@
-- V28: typed Platform API key authority reference of a durable Task (Story 4.3).
-- Only the immutable key identifier is stored, never the bearer secret.
alter table tasks add column if not exists platform_api_key_id text;
alter table tasks add constraint tasks_platform_api_key_id_check check (
platform_api_key_id is null
or octet_length(platform_api_key_id) between 1 and 160
);
@@ -0,0 +1,45 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "durable-tasks-v29";
pub(super) const SOURCE: &str = include_str!("durable_tasks_v29.sql");
pub(super) const SOURCE_SHA256: &str =
"0666c7d2aa440e6aae2170ceaf2fce2549118612e1505ef78817be60436a75dd";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v29",
Some(29),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v29",
Some(29),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -0,0 +1,52 @@
-- V29: durable per-outbound attempt history, pinned retry policy and cancellation (Story 4.4).
-- An accepted Task keeps the retry snapshot it was admitted with; legacy Tasks get a single attempt.
alter table tasks add column if not exists max_attempts integer not null default 1;
alter table tasks add column if not exists retry_backoff_step_ms bigint not null default 0;
alter table tasks add column if not exists next_attempt_at timestamptz;
alter table tasks add column if not exists cancel_requested_at timestamptz;
alter table tasks add column if not exists cancel_reason text;
alter table tasks add constraint tasks_max_attempts_check check (max_attempts between 1 and 5);
alter table tasks add constraint tasks_retry_backoff_step_check
check (retry_backoff_step_ms between 0 and 1000);
alter table tasks add constraint tasks_cancel_reason_check check (
cancel_reason is null or octet_length(cancel_reason) between 1 and 160
);
-- One row per committed outbound attempt. The authoritative outcome of a Task stays on `tasks`;
-- this history only records what was attempted and how each attempt resolved.
create table if not exists task_attempts (
attempt_id text primary key,
task_id uuid not null references tasks (task_id) on delete cascade,
ordinal integer not null,
owner text not null,
fence bigint not null default 0,
input_fingerprint text not null,
upstream_key_digest text,
retry_allowed boolean not null default false,
checkpoint_at timestamptz not null default now(),
resolved_at timestamptz,
outcome text,
certainty text,
phase text,
constraint task_attempts_task_ordinal_unique unique (task_id, ordinal),
constraint task_attempts_ordinal_check check (ordinal between 1 and 5),
constraint task_attempts_outcome_check check (
outcome is null
or outcome in ('completed', 'failed', 'retry_scheduled', 'outcome_unknown')
),
constraint task_attempts_certainty_check check (
certainty is null or certainty in ('known', 'unknown')
),
constraint task_attempts_phase_check check (
phase is null or phase in ('before_dispatch', 'dispatched')
),
-- A resolved attempt is complete: an unresolved row carries no resolution at all, so a stale
-- writer can never append a second outcome to the same attempt.
constraint task_attempts_resolution_check check (
(resolved_at is null and outcome is null and certainty is null and phase is null)
or (resolved_at is not null and outcome is not null and certainty is not null)
)
);
create index if not exists task_attempts_task_id_idx on task_attempts (task_id, ordinal);
@@ -480,8 +480,79 @@ pub(super) async fn validate_schema_fingerprint(
if !super::schema_guard_v25::validate_v25_absent(connection).await? {
return Err(schema_error(current_version));
}
} else {
if !super::schema_guard_v26::validate_v26_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v27::validate_v27_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 26 {
super::schema_guard_v25::validate_v25_durable_tasks(connection).await?;
if !super::schema_guard_v26::validate_v26_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 27 {
super::schema_guard_v26::validate_v26_durable_tasks(connection).await?;
if !super::schema_guard_v27::validate_v27_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 28 {
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
super::schema_guard_v27::validate_v27_durable_tasks(connection).await?;
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 29 {
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
super::schema_guard_v28::validate_v28_durable_tasks(connection).await?;
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 30 {
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
super::schema_guard_v29::validate_v29_durable_attempts(connection).await?;
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
return Err(schema_error(current_version));
}
} else {
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
super::schema_guard_v30::validate_v30_input_continuation(connection).await?;
}
Ok(())
}
@@ -0,0 +1,120 @@
//! V26 schema guard: lease/fencing columns for durable background Tasks (Story 4.3).
use sqlx::{PgConnection, Row, query};
use super::authority::MigrationError;
use super::schema_guard::relation_exists;
const TASKS: &str = "tasks";
const TASK_COLUMNS: &[(&str, &str, bool)] = &[
("task_id", "uuid", false),
("workspace_id", "text", false),
("owner_kind", "text", false),
("owner_id", "text", false),
("authorization_scope", "text", false),
("operation_id", "text", false),
("operation_version", "integer", false),
("scoped_key", "text", false),
("input_fingerprint", "text", false),
("request_id", "text", false),
("trace_id", "text", false),
("traceparent", "text", false),
("trace_sampled", "boolean", false),
("policy_mode", "text", false),
("approval_reference", "text", true),
("state", "text", false),
("version", "integer", false),
("attempts", "integer", false),
("input_envelope", "jsonb", false),
("progress_envelope", "jsonb", true),
("result_envelope", "jsonb", true),
("error_envelope", "jsonb", true),
("execution_deadline", "timestamp with time zone", false),
("retain_until", "timestamp with time zone", false),
("created_at", "timestamp with time zone", false),
("updated_at", "timestamp with time zone", false),
("fence", "bigint", false),
("lease_owner", "text", true),
("lease_until", "timestamp with time zone", true),
("current_attempt_id", "text", true),
("dispatch_started_at", "timestamp with time zone", true),
];
fn v26_error(stage: &'static str) -> MigrationError {
MigrationError::new(
"partial_sequence",
stage,
Some(26),
"restore_known_good_backup",
)
}
pub(super) async fn validate_v26_absent(
connection: &mut PgConnection,
) -> Result<bool, MigrationError> {
if relation_exists(connection, TASKS).await? {
// delegate to v25 absent for relation mostly; V26 columns must not exist.
for column in [
"fence",
"lease_owner",
"lease_until",
"current_attempt_id",
"dispatch_started_at",
] {
if super::schema_guard::column_exists(connection, TASKS, column).await? {
return Ok(false);
}
}
}
Ok(true)
}
pub(super) async fn validate_v26_base_columns(
connection: &mut PgConnection,
) -> Result<(), MigrationError> {
if !relation_exists(connection, TASKS).await? {
return Err(v26_error("v26.relations"));
}
for (column, data_type, nullable) in TASK_COLUMNS {
let row = query(
"select data_type, is_nullable from information_schema.columns
where table_schema = current_schema() and table_name = $1 and column_name = $2",
)
.bind(TASKS)
.bind(column)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
let valid = row.is_some_and(|row| {
row.try_get::<String, _>("data_type").ok().as_deref() == Some(*data_type)
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
== Some(if *nullable { "YES" } else { "NO" })
});
if !valid {
return Err(v26_error("v26.columns"));
}
}
Ok(())
}
pub(super) async fn validate_v26_durable_tasks(
connection: &mut PgConnection,
) -> Result<(), MigrationError> {
validate_v26_base_columns(connection).await?;
let count: i64 = query(
"select count(*)::bigint as count from information_schema.columns
where table_schema = current_schema() and table_name = $1",
)
.bind(TASKS)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != i64::try_from(TASK_COLUMNS.len()).unwrap_or_default() {
return Err(v26_error("v26.columns"));
}
// Constraints/index checks inherited conceptually from V25; V26 adds columns only.
Ok(())
}
@@ -0,0 +1,85 @@
//! V27 schema guard: encrypted executable Task input columns (Story 4.3).
use sqlx::{PgConnection, Row, query};
use super::authority::MigrationError;
use super::schema_guard::relation_exists;
use super::schema_guard_v26::validate_v26_base_columns;
const TASKS: &str = "tasks";
pub(super) async fn validate_v27_absent(
connection: &mut PgConnection,
) -> Result<bool, MigrationError> {
if relation_exists(connection, TASKS).await? {
for column in [
"executable_input_ciphertext",
"executable_input_key_version",
"executable_input_master_key_epoch",
] {
if super::schema_guard::column_exists(connection, TASKS, column).await? {
return Ok(false);
}
}
}
Ok(true)
}
pub(super) async fn validate_v27_durable_tasks(
connection: &mut PgConnection,
) -> Result<(), MigrationError> {
validate_v26_base_columns(connection).await?;
// Additional V27 columns.
let extra = [
("executable_input_ciphertext", "text", true),
("executable_input_key_version", "text", true),
("executable_input_master_key_epoch", "bigint", true),
("target_executable_input_ciphertext", "text", true),
("target_executable_input_key_version", "text", true),
("target_executable_input_master_key_epoch", "bigint", true),
];
for (column, data_type, nullable) in extra {
let row = query(
"select data_type, is_nullable from information_schema.columns
where table_schema = current_schema() and table_name = $1 and column_name = $2",
)
.bind(TASKS)
.bind(column)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
let valid = row.is_some_and(|row| {
row.try_get::<String, _>("data_type").ok().as_deref() == Some(data_type)
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
== Some(if nullable { "YES" } else { "NO" })
});
if !valid {
return Err(MigrationError::new(
"partial_sequence",
"v27.columns",
Some(27),
"restore_known_good_backup",
));
}
}
// Exact total column count for version 27: 43 V26 columns + 3 V27 columns.
let count: i64 = query(
"select count(*)::bigint as count from information_schema.columns
where table_schema = current_schema() and table_name = $1",
)
.bind(TASKS)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != 37_i64 {
return Err(MigrationError::new(
"partial_sequence",
"v27.columns",
Some(27),
"restore_known_good_backup",
));
}
Ok(())
}
@@ -0,0 +1,89 @@
//! V28 schema guard: typed Platform API key authority reference of a durable Task (Story 4.3).
use sqlx::{PgConnection, Row, query};
use super::authority::MigrationError;
use super::schema_guard::relation_exists;
const TASKS: &str = "tasks";
pub(super) async fn validate_v28_absent(
connection: &mut PgConnection,
) -> Result<bool, MigrationError> {
if relation_exists(connection, TASKS).await?
&& super::schema_guard::column_exists(connection, TASKS, "platform_api_key_id").await?
{
return Ok(false);
}
Ok(true)
}
pub(super) async fn validate_v28_durable_tasks(
connection: &mut PgConnection,
) -> Result<(), MigrationError> {
// The V27 column set is still required and unchanged; V28 only appends one column.
validate_v27_base_columns(connection).await?;
let row = query(
"select data_type, is_nullable from information_schema.columns
where table_schema = current_schema() and table_name = $1 and column_name = $2",
)
.bind(TASKS)
.bind("platform_api_key_id")
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
let valid = row.is_some_and(|row| {
row.try_get::<String, _>("data_type").ok().as_deref() == Some("text")
&& row.try_get::<String, _>("is_nullable").ok().as_deref() == Some("YES")
});
if !valid {
return Err(MigrationError::new(
"partial_sequence",
"v28.columns",
Some(28),
"restore_known_good_backup",
));
}
// Exact total column count for version 28: the 37 V27 columns + 1 V28 column.
let count: i64 = query(
"select count(*)::bigint as count from information_schema.columns
where table_schema = current_schema() and table_name = $1",
)
.bind(TASKS)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != 38_i64 {
return Err(MigrationError::new(
"partial_sequence",
"v28.columns",
Some(28),
"restore_known_good_backup",
));
}
Ok(())
}
/// V27 columns must still be exactly present before V28 is considered applied.
async fn validate_v27_base_columns(connection: &mut PgConnection) -> Result<(), MigrationError> {
for column in [
"executable_input_ciphertext",
"executable_input_key_version",
"executable_input_master_key_epoch",
"target_executable_input_ciphertext",
"target_executable_input_key_version",
"target_executable_input_master_key_epoch",
] {
if !super::schema_guard::column_exists(connection, TASKS, column).await? {
return Err(MigrationError::new(
"partial_sequence",
"v28.columns",
Some(28),
"restore_known_good_backup",
));
}
}
Ok(())
}
@@ -0,0 +1,87 @@
//! V29 schema guard: durable per-outbound attempt history, pinned retry policy and cancellation
//! (Story 4.4).
use sqlx::{PgConnection, Row, query};
use super::authority::MigrationError;
use super::schema_guard::{column_exists, relation_exists};
const TASKS: &str = "tasks";
const ATTEMPTS: &str = "task_attempts";
pub(super) async fn validate_v29_absent(
connection: &mut PgConnection,
) -> Result<bool, MigrationError> {
if relation_exists(connection, TASKS).await? {
if column_exists(connection, TASKS, "max_attempts").await? {
return Ok(false);
}
if relation_exists(connection, ATTEMPTS).await? {
return Ok(false);
}
}
Ok(true)
}
pub(super) async fn validate_v29_durable_attempts(
connection: &mut PgConnection,
) -> Result<(), MigrationError> {
// The V28 column set is still required and unchanged; V29 only appends columns and one table.
for column in [
"platform_api_key_id",
"max_attempts",
"retry_backoff_step_ms",
"next_attempt_at",
"cancel_requested_at",
"cancel_reason",
] {
if !column_exists(connection, TASKS, column).await? {
return Err(invalid(29, "v29.columns"));
}
}
// Exact total column count for version 29: the 38 V28 columns + 5 V29 columns.
let count: i64 = query(
"select count(*)::bigint as count from information_schema.columns
where table_schema = current_schema() and table_name = $1",
)
.bind(TASKS)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != 43_i64 {
return Err(invalid(29, "v29.columns"));
}
if !relation_exists(connection, ATTEMPTS).await? {
return Err(invalid(29, "v29.attempts"));
}
for column in [
"attempt_id",
"task_id",
"ordinal",
"owner",
"fence",
"input_fingerprint",
"upstream_key_digest",
"retry_allowed",
"checkpoint_at",
"resolved_at",
"outcome",
"certainty",
"phase",
] {
if !column_exists(connection, ATTEMPTS, column).await? {
return Err(invalid(29, "v29.attempts"));
}
}
Ok(())
}
fn invalid(version: i64, stage: &'static str) -> MigrationError {
MigrationError::new(
"partial_sequence",
stage,
Some(version),
"restore_known_good_backup",
)
}
@@ -0,0 +1,194 @@
//! V30 schema guard: continuation through validated input (Story 4.5).
use sqlx::{PgConnection, Row, query};
use super::authority::MigrationError;
use super::schema_guard::{column_exists, relation_exists};
pub(super) async fn validate_v30_absent(
connection: &mut PgConnection,
) -> Result<bool, MigrationError> {
if relation_exists(connection, "tasks").await?
&& column_exists(connection, "tasks", "effective_input_fingerprint").await?
{
return Ok(false);
}
if relation_exists(connection, "task_input_requests").await? {
return Ok(false);
}
if relation_exists(connection, "task_input_audit").await? {
return Ok(false);
}
Ok(true)
}
pub(super) async fn validate_v30_input_continuation(
connection: &mut PgConnection,
) -> Result<(), MigrationError> {
// V30 builds on the applied V29 shape. The exact V29 column count (43) no longer holds once V30
// appends its column (44), so the V29-specific columns are re-checked here by name instead of
// calling the exact-count V29 guard, which is only valid before V30 is applied.
for column in [
"platform_api_key_id",
"max_attempts",
"retry_backoff_step_ms",
"next_attempt_at",
"cancel_requested_at",
"cancel_reason",
] {
if !column_exists(connection, "tasks", column).await? {
return Err(invalid(30, "v30.columns"));
}
}
// V30 appends one column to tasks.
if !column_exists(connection, "tasks", "effective_input_fingerprint").await? {
return Err(invalid(30, "v30.columns"));
}
let count: i64 = query(
"select count(*)::bigint as count from information_schema.columns
where table_schema = current_schema() and table_name = $1",
)
.bind("tasks")
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != 44_i64 {
return Err(invalid(30, "v30.columns"));
}
// Tables.
for table in ["task_input_requests", "task_input_audit"] {
if !relation_exists(connection, table).await? {
return Err(invalid(30, "v30.tables"));
}
}
// task_input_requests columns.
for column in [
"task_id",
"wait_id",
"expected_version",
"schema_revision",
"schema_format_version",
"fingerprint_format_version",
"schema_snapshot",
"projected_schema",
"prompt_code",
"missing_fields",
"expires_at",
"created_at",
"accepted_at",
"update_key_digest",
"canonical_fingerprint",
"effective_input_fingerprint",
"committed_state",
"committed_version",
] {
if !column_exists(connection, "task_input_requests", column).await? {
return Err(invalid(30, "v30.task_input_requests"));
}
}
// task_input_audit columns.
for column in [
"id",
"task_id",
"wait_id",
"event",
"from_state",
"to_state",
"from_version",
"to_version",
"actor",
"reason_code",
"created_at",
] {
if !column_exists(connection, "task_input_audit", column).await? {
return Err(invalid(30, "v30.task_input_audit"));
}
}
// Constraints.
for constraint in [
"task_input_requests_pkey",
"task_input_requests_wait_unique",
"task_input_requests_prompt_check",
"task_input_requests_receipt_check",
"task_input_audit_event_check",
"task_input_audit_actor_check",
"task_input_audit_reason_check",
"tasks_effective_input_fingerprint_check",
] {
let table = if constraint.starts_with("task_input_audit")
|| constraint.starts_with("task_input_audit_")
{
"task_input_audit"
} else if constraint == "tasks_effective_input_fingerprint_check" {
"tasks"
} else {
"task_input_requests"
};
if !named_constraint_exists(connection, table, constraint).await? {
return Err(invalid(30, "v30.constraints"));
}
}
// Indexes.
for index in [
"task_input_requests_active_ttl_idx",
"task_input_audit_task_idx",
] {
let present = query(
"select exists (select 1 from pg_catalog.pg_indexes
where schemaname = current_schema() and indexname = $1) as present",
)
.bind(index)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get::<bool, _>("present")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if !present {
return Err(invalid(30, "v30.indexes"));
}
}
Ok(())
}
fn invalid(version: i64, stage: &'static str) -> MigrationError {
MigrationError::new(
"partial_sequence",
stage,
Some(version),
"restore_known_good_backup",
)
}
// helper
async fn named_constraint_exists(
connection: &mut PgConnection,
table: &str,
constraint: &str,
) -> Result<bool, MigrationError> {
query(
"select exists (
select 1 from pg_catalog.pg_constraint c
join pg_catalog.pg_class t on t.oid = c.conrelid
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
where n.nspname = current_schema()
and t.relname = $1
and c.conname = $2
) as present",
)
.bind(table)
.bind(constraint)
.fetch_one(connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get::<bool, _>("present")
.map_err(|_| MigrationError::storage("preflight.schema"))
}
@@ -577,6 +577,145 @@ pub(super) async fn insert_admission_entry(
Ok(u64::try_from(ticket_id).unwrap_or_default())
}
/// Shared grant step used by the durable Task store: grants one reserved Task intent if capacity allows,
/// marks it `executing`, persists the weighted round-robin cursor, and returns its Task id. It reuses
/// the same selector/capacity logic as synchronous admission so no second policy exists.
pub(super) async fn grant_next_task(
transaction: &mut Transaction<'_, Postgres>,
workspace_id: &str,
cursor: &AdmissionCursor,
limits: &AdmissionLimits,
scheduler: AdmissionSchedulerConfig,
) -> Result<(Option<Uuid>, AdmissionCursor), AttemptError> {
let rows = sqlx::query(
"select ticket_id, agent_key, operation_id, task_id
from execution_admission_entries
where state = 'reserved' and task_id is not null
order by ticket_id
limit $1
for update skip locked",
)
.bind(QUEUE_READ_LIMIT)
.fetch_all(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
if rows.is_empty() {
return Ok((None, cursor.clone()));
}
struct Candidate {
entry: AdmissionQueueEntry,
task_id: Uuid,
}
let mut candidates = Vec::with_capacity(rows.len());
for row in rows {
let ticket_id: i64 = row.try_get("ticket_id").map_err(AttemptError::from_sql)?;
let task_id: Uuid = row.try_get("task_id").map_err(AttemptError::from_sql)?;
candidates.push(Candidate {
entry: AdmissionQueueEntry {
ticket_id: u64::try_from(ticket_id).unwrap_or_default(),
sequence: 0,
agent_key: row.try_get("agent_key").map_err(AttemptError::from_sql)?,
operation_id: row
.try_get("operation_id")
.map_err(AttemptError::from_sql)?,
},
task_id,
});
}
let queue: Vec<AdmissionQueueEntry> = candidates.iter().map(|c| c.entry.clone()).collect();
let Some(selection) = select_next_grant(&queue, cursor, scheduler) else {
return Ok((None, cursor.clone()));
};
let head_index = selection.index;
let head = &candidates[head_index];
let global_executing: i64 = sqlx::query(
"select count(*)::bigint as count from execution_admission_entries where state = 'executing'",
)
.fetch_one(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?
.try_get("count")
.map_err(AttemptError::from_sql)?;
let agent_counts = sqlx::query(
"select agent_key as key, count(*)::bigint as count
from execution_admission_entries where state = 'executing' group by agent_key",
)
.fetch_all(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let mut agent_map = std::collections::BTreeMap::new();
for row in agent_counts {
let key: String = row.try_get("key").map_err(AttemptError::from_sql)?;
let count: i64 = row.try_get("count").map_err(AttemptError::from_sql)?;
agent_map.insert(key, u64::try_from(count).unwrap_or_default());
}
let tool_counts = sqlx::query(
"select operation_id as key, count(*)::bigint as count
from execution_admission_entries where state = 'executing' group by operation_id",
)
.fetch_all(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let mut tool_map = std::collections::BTreeMap::new();
for row in tool_counts {
let key: String = row.try_get("key").map_err(AttemptError::from_sql)?;
let count: i64 = row.try_get("count").map_err(AttemptError::from_sql)?;
tool_map.insert(key, u64::try_from(count).unwrap_or_default());
}
if u64::try_from(global_executing).unwrap_or(u64::MAX)
>= u64::try_from(limits.global_limit).unwrap_or(u64::MAX)
{
return Ok((None, cursor.clone()));
}
let agent_executing = agent_map
.get(head.entry.agent_key.as_str())
.copied()
.unwrap_or_default();
if agent_executing >= u64::try_from(limits.agent_limit).unwrap_or(u64::MAX) {
return Ok((None, cursor.clone()));
}
let tool_executing = tool_map
.get(head.entry.operation_id.as_str())
.copied()
.unwrap_or_default();
if tool_executing >= u64::try_from(limits.tool_limit).unwrap_or(u64::MAX) {
return Ok((None, cursor.clone()));
}
sqlx::query(
"update execution_admission_entries
set state = 'executing', granted_at = clock_timestamp()
where ticket_id = $1 and state = 'reserved'",
)
.bind(i64::try_from(head.entry.ticket_id).unwrap_or(i64::MAX))
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
sqlx::query(
"update execution_admission_controllers
set cursor_agent = $2,
cursor_agent_credit = $3,
cursor_tool = $4,
cursor_tool_credit = $5,
updated_at = now()
where workspace_id = $1",
)
.bind(workspace_id)
.bind(selection.cursor.agent.as_deref())
.bind(i32::try_from(selection.cursor.agent_credit).unwrap_or_default())
.bind(selection.cursor.tool.as_deref())
.bind(i32::try_from(selection.cursor.tool_credit).unwrap_or_default())
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
Ok((Some(head.task_id), selection.cursor))
}
#[derive(Debug)]
pub(super) enum AttemptError {
/// Lock or statement timeout: the transaction made no decision and the waiter retries.
Contended,
@@ -170,6 +170,69 @@ impl PostgresRegistry {
.transpose()
}
/// Authoritative revalidation of the Platform API key that authorized a durable Task (Story 4.3).
///
/// The key is returned only when the exact immutable ID is still active, unexpired and bound to
/// the same workspace and agent; otherwise `None`, so a worker fails closed. Only the immutable
/// identity is supplied here — the bearer secret is never stored or re-read.
pub async fn revalidate_platform_api_key(
&self,
workspace_id: &WorkspaceId,
agent_id: &AgentId,
key_id: &PlatformApiKeyId,
) -> Result<Option<PlatformApiKeyRecord>, RegistryError> {
let row = sqlx::query(
"select
k.id,
k.workspace_id,
k.agent_id,
k.key_kind,
k.name,
k.prefix,
k.scopes_json,
k.status,
k.created_at,
k.last_used_at,
k.expires_at,
k.allowed_origins_json
from platform_api_keys k
where k.id = $1
and k.workspace_id = $2
and k.agent_id = $3
and k.key_kind = 'mcp_client'
and k.status = 'active'
and (k.expires_at is null or k.expires_at > now())
limit 1",
)
.bind(key_id.as_str())
.bind(workspace_id.as_str())
.bind(agent_id.as_str())
.fetch_optional(&self.pool)
.await?;
row.map(|row| {
Ok(PlatformApiKeyRecord {
api_key: PlatformApiKey {
id: PlatformApiKeyId::new(row.get::<String, _>("id")),
workspace_id: WorkspaceId::new(row.get::<String, _>("workspace_id")),
agent_id: row.get::<Option<String>, _>("agent_id").map(AgentId::new),
key_kind: deserialize_enum_text(&row.get::<String, _>("key_kind"), "key_kind")?,
name: row.get("name"),
prefix: row.get("prefix"),
scopes: deserialize_json_value(row.get::<Value, _>("scopes_json"))?,
status: deserialize_enum_text(&row.get::<String, _>("status"), "status")?,
created_at: row.get("created_at"),
last_used_at: row.get("last_used_at"),
expires_at: row.get("expires_at"),
allowed_origins: deserialize_json_value(
row.get::<Value, _>("allowed_origins_json"),
)?,
},
})
})
.transpose()
}
pub async fn get_approval_api_key_by_secret_for_agent_slug(
&self,
workspace_slug: &str,
@@ -1,6 +1,6 @@
use super::*;
const MASTER_KEY_PAGE_LIMIT_MAX: i64 = 10_000;
pub(super) const MASTER_KEY_PAGE_LIMIT_MAX: i64 = 10_000;
impl PostgresRegistry {
pub async fn verify_or_register_master_key_identity(
@@ -356,12 +356,19 @@ impl PostgresRegistry {
return Err(RegistryError::MasterKeyRotationConflict);
}
let total = sqlx::query_scalar::<_, i64>(
let secret_total = sqlx::query_scalar::<_, i64>(
"select count(*) from secret_versions where master_key_epoch = $1",
)
.bind(source_epoch)
.fetch_one(&mut *tx)
.await?;
let task_total = sqlx::query_scalar::<_, i64>(
"select count(*) from tasks where executable_input_master_key_epoch = $1",
)
.bind(source_epoch)
.fetch_one(&mut *tx)
.await?;
let total = secret_total + task_total;
let rotation_id = format!("master-key-e{source_epoch}-to-e{target_epoch}");
let aborted = sqlx::query(
"select
@@ -403,6 +410,16 @@ impl PostgresRegistry {
.bind(target_epoch)
.execute(&mut *tx)
.await?;
sqlx::query(
"update tasks
set target_executable_input_ciphertext = null,
target_executable_input_key_version = null,
target_executable_input_master_key_epoch = null
where target_executable_input_master_key_epoch = $1",
)
.bind(target_epoch)
.execute(&mut *tx)
.await?;
sqlx::query(
"update master_key_rotations
set state = 'running',
@@ -662,9 +679,24 @@ impl PostgresRegistry {
.bind(rotation.target_epoch)
.execute(&mut *tx)
.await?;
sqlx::query(
"update tasks
set executable_input_ciphertext = target_executable_input_ciphertext,
executable_input_key_version = target_executable_input_key_version,
executable_input_master_key_epoch = target_executable_input_master_key_epoch,
target_executable_input_ciphertext = null,
target_executable_input_key_version = null,
target_executable_input_master_key_epoch = null
where executable_input_master_key_epoch = $1
and target_executable_input_master_key_epoch = $2",
)
.bind(rotation.source_epoch)
.bind(rotation.target_epoch)
.execute(&mut *tx)
.await?;
sqlx::query(
"update master_key_rotations
set state = 'promoted',
set state = 'promoted',
updated_at = $2::timestamptz
where id = $1",
)
@@ -697,6 +729,16 @@ impl PostgresRegistry {
.bind(rotation.target_epoch)
.execute(&mut *tx)
.await?;
sqlx::query(
"update tasks
set target_executable_input_ciphertext = null,
target_executable_input_key_version = null,
target_executable_input_master_key_epoch = null
where target_executable_input_master_key_epoch = $1",
)
.bind(rotation.target_epoch)
.execute(&mut *tx)
.await?;
sqlx::query(
"update master_key_rotations
set state = 'aborted',
@@ -883,7 +925,7 @@ async fn select_active_master_key_identity_for_update(
Ok(row.map(map_master_key_identity))
}
async fn select_master_key_rotation_for_update(
pub(super) async fn select_master_key_rotation_for_update(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
rotation_id: &str,
) -> Result<MasterKeyRotationRecord, RegistryError> {
@@ -915,16 +957,19 @@ async fn select_master_key_rotation_for_update(
})
}
async fn count_staged_rotation_targets(
pub(super) async fn count_staged_rotation_targets(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
source_epoch: i64,
target_epoch: i64,
) -> Result<i64, RegistryError> {
let count = sqlx::query_scalar::<_, i64>(
"select count(*)
from secret_versions
where master_key_epoch = $1
and target_master_key_epoch = $2",
"select
(select count(*) from secret_versions
where master_key_epoch = $1
and target_master_key_epoch = $2) +
(select count(*) from tasks
where executable_input_master_key_epoch = $1
and target_executable_input_master_key_epoch = $2)",
)
.bind(source_epoch)
.bind(target_epoch)
@@ -0,0 +1,144 @@
//! Task ciphertext paging/staging of the master-key rotation (Story 4.3).
//!
//! Extracted from `master_key.rs` without logic changes so that module stays within the
//! code-health line limit; the operations and their SQL are unchanged.
use super::master_key::{
MASTER_KEY_PAGE_LIMIT_MAX, count_staged_rotation_targets, select_master_key_rotation_for_update,
};
use super::*;
fn map_master_key_task_version(
row: sqlx::postgres::PgRow,
) -> Result<MasterKeyTaskVersion, RegistryError> {
Ok(MasterKeyTaskVersion {
task_id: row.get::<String, _>("task_id"),
ciphertext: row
.get::<Option<String>, _>("executable_input_ciphertext")
.unwrap_or_default(),
key_version: row
.get::<Option<String>, _>("executable_input_key_version")
.unwrap_or_default(),
epoch: row
.get::<Option<i64>, _>("executable_input_master_key_epoch")
.unwrap_or_default(),
target_ciphertext: row.get::<Option<String>, _>("target_executable_input_ciphertext"),
target_key_version: row.get::<Option<String>, _>("target_executable_input_key_version"),
target_epoch: row.get::<Option<i64>, _>("target_executable_input_master_key_epoch"),
})
}
impl PostgresRegistry {
pub async fn list_tasks_for_master_key_epoch_page(
&self,
epoch: i64,
after_task_id: Option<&str>,
limit: i64,
) -> Result<Vec<MasterKeyTaskVersion>, RegistryError> {
if epoch < 1 || !(1..=MASTER_KEY_PAGE_LIMIT_MAX).contains(&limit) {
return Err(RegistryError::InvalidMasterKeyIdentity);
}
let rows = sqlx::query(
"select task_id::text as task_id, executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch, target_executable_input_ciphertext, target_executable_input_key_version, target_executable_input_master_key_epoch from tasks where executable_input_master_key_epoch = $1 and ($2::text is null or task_id::text > $2::text) order by task_id asc limit $3",
)
.bind(epoch)
.bind(after_task_id)
.bind(limit)
.fetch_all(&self.pool)
.await?;
rows.into_iter().map(map_master_key_task_version).collect()
}
pub async fn list_target_tasks_for_master_key_rotation_page(
&self,
target_epoch: i64,
after_task_id: Option<&str>,
limit: i64,
) -> Result<Vec<MasterKeyTaskVersion>, RegistryError> {
if target_epoch < 1 || !(1..=MASTER_KEY_PAGE_LIMIT_MAX).contains(&limit) {
return Err(RegistryError::InvalidMasterKeyIdentity);
}
let rows = sqlx::query(
"select task_id::text as task_id, executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch, target_executable_input_ciphertext, target_executable_input_key_version, target_executable_input_master_key_epoch from tasks where target_executable_input_master_key_epoch = $1 and ($2::text is null or task_id::text > $2::text) order by task_id asc limit $3",
)
.bind(target_epoch)
.bind(after_task_id)
.bind(limit)
.fetch_all(&self.pool)
.await?;
rows.into_iter().map(map_master_key_task_version).collect()
}
#[allow(clippy::too_many_arguments)]
pub async fn stage_master_key_rotation_task_ciphertext(
&self,
rotation_id: &str,
task_id: &str,
source_epoch: i64,
target_ciphertext: &str,
target_key_version: &str,
target_epoch: i64,
now: &OffsetDateTime,
) -> Result<MasterKeyRotationRecord, RegistryError> {
let mut tx = self.pool.begin().await?;
let rotation = select_master_key_rotation_for_update(&mut tx, rotation_id).await?;
if rotation.state != "running"
|| rotation.source_epoch != source_epoch
|| rotation.target_epoch != target_epoch
{
return Err(RegistryError::MasterKeyRotationConflict);
}
let result = sqlx::query(
"update tasks
set target_executable_input_ciphertext = $3,
target_executable_input_key_version = $4,
target_executable_input_master_key_epoch = $5
where task_id::text = $1
and executable_input_master_key_epoch = $2
and (
target_executable_input_master_key_epoch is null
or target_executable_input_master_key_epoch = $5
)
and (
target_executable_input_ciphertext is null
or target_executable_input_ciphertext = $3
)",
)
.bind(task_id)
.bind(source_epoch)
.bind(target_ciphertext)
.bind(target_key_version)
.bind(target_epoch)
.execute(&mut *tx)
.await?;
if result.rows_affected() != 1 {
return Err(RegistryError::MasterKeyRotationConflict);
}
let processed = count_staged_rotation_targets(&mut tx, source_epoch, target_epoch).await?;
sqlx::query(
"update master_key_rotations
set checkpoint_secret_id = checkpoint_secret_id,
processed_secret_versions = $2,
updated_at = $3::timestamptz
where id = $1",
)
.bind(rotation_id)
.bind(processed)
.bind(now)
.execute(&mut *tx)
.await?;
let updated = select_master_key_rotation_for_update(&mut tx, rotation_id).await?;
tx.commit().await?;
Ok(updated)
}
}
pub struct MasterKeyTaskVersion {
pub task_id: String,
pub ciphertext: String,
pub key_version: String,
pub epoch: i64,
pub target_ciphertext: Option<String>,
pub target_key_version: Option<String>,
pub target_epoch: Option<i64>,
}
@@ -10,6 +10,7 @@ mod auth;
mod connection;
mod import_job;
mod master_key;
mod master_key_tasks;
mod observability;
mod onboarding;
mod onboarding_import;
-586
View File
@@ -1,586 +0,0 @@
//! PostgreSQL implementation of the durable Task store (Story 4.2, FR-22/FR-23).
//!
//! One PostgreSQL transaction owns the whole acceptance: Task record, dispatch intent
//! (`execution_admission_entries.task_id`) and its queue slot commit together, so no Task ID and no
//! acknowledgement can exist without the durable record. The transaction reuses the transaction-aware
//! primitives of the execution-admission store (`lock_admission_in_tx`, `bounded_queue_len`,
//! `insert_admission_entry`) instead of copying capacity SQL or starting a nested transaction.
//!
//! Replay is resolved before any capacity mutation and never touches the accepted row. An ambiguous
//! COMMIT is reported as an unavailable store: it is never treated as a proven rollback, so no
//! compensating delete and no synchronous fallback can happen after an uncertain acceptance.
use async_trait::async_trait;
use crank_core::{
AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence, NoTasksFallback,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest,
TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskStore,
TaskSupportIds, ToolExecutionMode,
};
use serde_json::Value;
use sqlx::{PgPool, Postgres, Row, Transaction};
use uuid::Uuid;
use super::admission::{
AdmissionLimits, AttemptError, bounded_queue_len, cleanup_expired_unlinked,
insert_admission_entry, lock_admission_in_tx,
};
const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_id,
authorization_scope, operation_id, input_fingerprint, operation_version, policy_mode,
approval_reference,
request_id, trace_id, traceparent, trace_sampled, input_envelope, progress_envelope,
result_envelope, error_envelope, attempts, version,
(extract(epoch from execution_deadline) * 1000)::bigint as execution_deadline_ms,
(extract(epoch from retain_until) * 1000)::bigint as retain_until_ms,
(extract(epoch from created_at) * 1000)::bigint as created_ms,
(extract(epoch from updated_at) * 1000)::bigint as updated_ms";
#[derive(Clone)]
pub struct PostgresTaskStore {
pool: PgPool,
limits: AdmissionLimits,
}
impl PostgresTaskStore {
pub fn new(
pool: PgPool,
limits: AdmissionLimits,
scheduler: crank_core::AdmissionSchedulerConfig,
) -> Result<Self, TaskError> {
limits
.validate()
.map_err(|error| TaskError::invalid(error.to_string()))?;
scheduler
.validate()
.map_err(|error| TaskError::invalid(error.to_string()))?;
Ok(Self { pool, limits })
}
fn unavailable(error: impl std::fmt::Display) -> TaskError {
TaskError::unavailable(format!("durable task store is unavailable: {error}"))
}
async fn start(&self) -> Result<Transaction<'_, Postgres>, TaskError> {
let mut transaction = self.pool.begin().await.map_err(Self::unavailable)?;
sqlx::query("set local lock_timeout = '3000ms'")
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
sqlx::query("set local statement_timeout = '5000ms'")
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
Ok(transaction)
}
/// Scoped lookup of an accepted Task. The full owner/authorization/Tool predicate is part of the
/// query, so a foreign row is never fetched.
async fn stored_acceptance(
&self,
transaction: &mut Transaction<'_, Postgres>,
request: &TaskAcceptRequest,
) -> Result<Option<(TaskAcceptance, String)>, AttemptError> {
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
)))
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(&request.scoped_key)
.fetch_optional(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let Some(row) = row else {
return Ok(None);
};
let record = record_from_row(&row).map_err(AttemptError::Failure)?;
Ok(Some((
acceptance_from_record(&record),
record.input_fingerprint,
)))
}
}
fn acceptance_from_record(record: &TaskRecord) -> TaskAcceptance {
TaskAcceptance {
task_id: record.task_id,
state: record.state,
operation_version: record.pins.operation_version,
support: record.support.clone(),
created_at_unix_ms: record.created_at_unix_ms,
retain_until_unix_ms: record.pins.retain_until_unix_ms,
replayed: true,
}
}
/// Readiness probe of the durable Task state: the table must be reachable and readable. A missing or
/// unreadable schema is a dependency failure, never a degraded-but-ready state.
pub async fn tasks_health(pool: &PgPool) -> Result<(), TaskError> {
sqlx::query("select count(*)::bigint from tasks where false")
.fetch_one(pool)
.await
.map_err(PostgresTaskStore::unavailable)?;
Ok(())
}
/// One transaction's decision of the acceptance path.
enum Accepted {
Created(TaskAcceptance),
Replayed(TaskAcceptance),
Conflict,
}
#[async_trait]
impl TaskStore for PostgresTaskStore {
async fn health(&self) -> Result<(), TaskError> {
sqlx::query("select count(*)::bigint from tasks where false")
.fetch_one(&self.pool)
.await
.map_err(Self::unavailable)?;
super::admission::admission_health(&self.pool)
.await
.map_err(|error| TaskError::unavailable(error.to_string()))?;
Ok(())
}
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
let outcome = self.accept_locked(&mut transaction, &request).await;
match outcome {
Ok(Accepted::Created(acceptance) | Accepted::Replayed(acceptance)) => {
// The acknowledgement of a created Task is only possible after a real COMMIT.
transaction.commit().await.map_err(Self::unavailable)?;
Ok(TaskAcceptance {
replayed: acceptance.replayed,
..acceptance
})
}
Ok(Accepted::Conflict) => {
transaction.rollback().await.ok();
Err(TaskError::FingerprintConflict)
}
Err(AttemptError::Overloaded) => {
transaction.rollback().await.ok();
Err(TaskError::Overloaded {
retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS,
})
}
Err(AttemptError::Contended) => {
transaction.rollback().await.ok();
Err(TaskError::unavailable(
"durable task acceptance could not take the admission locks",
))
}
Err(AttemptError::UniqueViolation) => {
// Roll back the aborted transaction, then reread the scoped key once outside it.
transaction.rollback().await.ok();
self.reread_after_unique(&request).await
}
Err(AttemptError::Failure(error)) => {
transaction.rollback().await.ok();
Err(error)
}
Err(AttemptError::AdmissionFailure(error)) => {
transaction.rollback().await.ok();
Err(TaskError::unavailable(error.to_string()))
}
}
}
async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result<TaskRecord, TaskError> {
access.validate()?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5"
)))
.bind(task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::NotFound);
};
record_from_row(&row).map_err(Self::unavailable)
}
async fn find(
&self,
access: &TaskAccess,
operation_id: &str,
scoped_key: &str,
) -> Result<Option<TaskRecord>, TaskError> {
access.validate()?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
)))
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.bind(operation_id)
.bind(scoped_key)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
row.map(|row| record_from_row(&row)).transpose()
}
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
let access = &request.access;
let update = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = $1,
result_envelope = $2,
error_envelope = $3,
version = version + 1,
updated_at = now()
where task_id = $4 and workspace_id = $5 and owner_kind = $6
and owner_id = $7 and authorization_scope = $8
and state = $9 and version = $10
returning {TASK_PROJECTION}"
)))
.bind(request.target_state.as_str())
.bind(request.result.as_ref().map(|value| value.value().clone()))
.bind(request.error.as_ref().map(|value| value.value().clone()))
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.bind(request.expected_state.as_str())
.bind(i64::from(request.expected_version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = update else {
transaction.rollback().await.ok();
return Err(self.classify_missed_finish(&request).await);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
// Terminal Task and its dispatch intent leave together: a half-done release is rolled back.
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(request.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(record)
}
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError> {
let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT));
let mut transaction = self.start().await?;
let expired = sqlx::query(
"with expired as (
update tasks
set state = 'failed',
error_envelope = jsonb_build_object('reason', 'execution_deadline_elapsed'),
version = version + 1,
updated_at = now()
where task_id in (
select task_id from tasks
where state = 'queued' and execution_deadline <= now()
order by task_id
limit $1
)
returning task_id
), released as (
delete from execution_admission_entries
where task_id in (select task_id from expired)
returning task_id
)
select (select count(*) from expired)::bigint as expired_queued",
)
.bind(batch)
.fetch_one(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let expired_queued: i64 = expired
.try_get("expired_queued")
.map_err(Self::unavailable)?;
let purged = sqlx::query(
"delete from tasks
where task_id in (
select task_id from tasks
where state in ('completed', 'failed', 'cancelled') and retain_until <= now()
order by task_id
limit $1
)",
)
.bind(batch)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(TaskPurgeReport {
expired_queued: u64::try_from(expired_queued).unwrap_or_default(),
expired_terminal: purged.rows_affected(),
})
}
}
impl PostgresTaskStore {
async fn accept_locked(
&self,
transaction: &mut Transaction<'_, Postgres>,
request: &TaskAcceptRequest,
) -> Result<Accepted, AttemptError> {
// Replay first: an already accepted Task is returned without capacity mutation, so it works
// with a full queue and after the original execution deadline while the record is retained.
if let Some((acceptance, fingerprint)) =
self.stored_acceptance(transaction, request).await?
{
return Ok(if fingerprint == request.input_fingerprint {
Accepted::Replayed(acceptance)
} else {
Accepted::Conflict
});
}
// Serialize with synchronous admission and any concurrent acceptance of the same scope.
lock_admission_in_tx(transaction, &request.workspace_id).await?;
// The competitor may have committed while this transaction waited for the controller lock.
if let Some((acceptance, fingerprint)) =
self.stored_acceptance(transaction, request).await?
{
return Ok(if fingerprint == request.input_fingerprint {
Accepted::Replayed(acceptance)
} else {
Accepted::Conflict
});
}
cleanup_expired_unlinked(transaction).await?;
let queued = bounded_queue_len(transaction).await?;
if queued >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) {
return Err(AttemptError::Overloaded);
}
let created_ms: i64 = sqlx::query(
"insert into tasks (
task_id, workspace_id, owner_kind, owner_id, authorization_scope, operation_id,
operation_version, scoped_key, input_fingerprint, request_id, trace_id, traceparent,
trace_sampled, policy_mode, approval_reference, state, input_envelope,
execution_deadline, retain_until
) values (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued',
$16, to_timestamp($17::double precision / 1000.0),
to_timestamp($18::double precision / 1000.0)
)
returning (extract(epoch from created_at) * 1000)::bigint as created_ms",
)
.bind(request.task_id.as_uuid())
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(i32::try_from(request.pins.operation_version).unwrap_or(i32::MAX))
.bind(&request.scoped_key)
.bind(&request.input_fingerprint)
.bind(request.support.request_id.as_str())
.bind(request.support.trace_id.as_str())
.bind(&request.support.traceparent)
.bind(request.support.trace_sampled)
.bind(policy_mode_str(request.pins.policy_mode))
.bind(request.pins.approval_reference.as_deref())
.bind(request.input.value().clone())
.bind(request.pins.execution_deadline_unix_ms)
.bind(request.pins.retain_until_unix_ms)
.fetch_one(&mut **transaction)
.await
.map_err(task_sql_error)?
.try_get("created_ms")
.map_err(AttemptError::from_sql)?;
let admission_request = AdmissionRequest {
scope: AdmissionScope {
workspace_id: request.workspace_id.clone(),
agent_key: request.owner_id.clone(),
operation_id: request.operation_id.clone(),
operation_version: request.pins.operation_version,
},
mode: AdmissionMode::Background,
policy_mode: request.pins.policy_mode,
no_tasks_fallback: NoTasksFallback::Unsupported,
// Never reached: the intent row is inserted by this transaction, not by `admit`.
approval: ApprovalEvidence::NotRequired,
fingerprint: request.input_fingerprint.clone(),
expires_at_unix_ms: request.pins.execution_deadline_unix_ms,
};
let owner_token = format!("task_{}", request.task_id);
insert_admission_entry(
transaction,
&admission_request,
&owner_token,
"reserved",
Some(request.task_id.as_uuid()),
)
.await?;
Ok(Accepted::Created(TaskAcceptance {
task_id: request.task_id,
state: TaskState::Queued,
operation_version: request.pins.operation_version,
support: request.support.clone(),
created_at_unix_ms: created_ms,
retain_until_unix_ms: request.pins.retain_until_unix_ms,
replayed: false,
}))
}
/// Bounded scoped reread after a foreign unique conflict aborted the transaction.
async fn reread_after_unique(
&self,
request: &TaskAcceptRequest,
) -> Result<TaskAcceptance, TaskError> {
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
)))
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(&request.scoped_key)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::unavailable(
"durable task acceptance ended with an unresolved unique conflict",
));
};
let record = record_from_row(&row)?;
if record.input_fingerprint != request.input_fingerprint {
return Err(TaskError::FingerprintConflict);
}
Ok(acceptance_from_record(&record))
}
/// A rejected conditional update is either a lost CAS race or a foreign/absent Task. Both are
/// reported without payload, and a foreign Task is indistinguishable from an absent one.
async fn classify_missed_finish(&self, request: &TaskFinishRequest) -> TaskError {
let access = &request.access;
match sqlx::query(
"select state, version from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5",
)
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
{
Ok(Some(_)) => TaskError::StateConflict,
Ok(None) => TaskError::NotFound,
Err(error) => Self::unavailable(error),
}
}
}
fn policy_mode_str(mode: ToolExecutionMode) -> &'static str {
match mode {
ToolExecutionMode::Synchronous => "synchronous",
ToolExecutionMode::Background => "background",
ToolExecutionMode::Adaptive => "adaptive",
}
}
fn task_sql_error(error: sqlx::Error) -> AttemptError {
if error
.as_database_error()
.and_then(|database| database.code())
.as_deref()
== Some("23505")
{
return AttemptError::UniqueViolation;
}
AttemptError::from_sql(error)
}
fn record_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskRecord, TaskError> {
let task_id: Uuid = row.try_get("task_id").map_err(unavailable_row)?;
let state: String = row.try_get("state").map_err(unavailable_row)?;
let owner_kind: String = row.try_get("owner_kind").map_err(unavailable_row)?;
let policy_mode: String = row.try_get("policy_mode").map_err(unavailable_row)?;
let trace_id: String = row.try_get("trace_id").map_err(unavailable_row)?;
let request_id: String = row.try_get("request_id").map_err(unavailable_row)?;
let envelope = |column: &str| -> Result<Option<TaskEnvelope>, TaskError> {
let value: Option<Value> = row.try_get(column).map_err(unavailable_row)?;
value.map(TaskEnvelope::from_persisted).transpose()
};
Ok(TaskRecord {
task_id: TaskId::parse(&task_id.to_string())?,
state: TaskState::parse(&state)?,
workspace_id: row.try_get("workspace_id").map_err(unavailable_row)?,
owner_kind: TaskOwnerKind::parse(&owner_kind)?,
owner_id: row.try_get("owner_id").map_err(unavailable_row)?,
authorization_scope: row
.try_get("authorization_scope")
.map_err(unavailable_row)?,
operation_id: row.try_get("operation_id").map_err(unavailable_row)?,
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
pins: TaskPins {
operation_version: u32::try_from(
row.try_get::<i32, _>("operation_version")
.map_err(unavailable_row)?,
)
.unwrap_or_default(),
policy_mode: match policy_mode.as_str() {
"background" => ToolExecutionMode::Background,
"adaptive" => ToolExecutionMode::Adaptive,
_ => ToolExecutionMode::Synchronous,
},
execution_deadline_unix_ms: row
.try_get("execution_deadline_ms")
.map_err(unavailable_row)?,
retain_until_unix_ms: row.try_get("retain_until_ms").map_err(unavailable_row)?,
approval_reference: row.try_get("approval_reference").map_err(unavailable_row)?,
},
support: TaskSupportIds {
request_id: crank_core::RequestId::parse(&request_id)
.map_err(|_| TaskError::unavailable("stored request id is invalid"))?,
trace_id: crank_core::TraceId::parse(&trace_id)
.map_err(|_| TaskError::unavailable("stored trace id is invalid"))?,
traceparent: row.try_get("traceparent").map_err(unavailable_row)?,
trace_sampled: row.try_get("trace_sampled").map_err(unavailable_row)?,
},
input: envelope("input_envelope")?
.ok_or_else(|| TaskError::unavailable("stored task input envelope is missing"))?,
progress: envelope("progress_envelope")?,
result: envelope("result_envelope")?,
error: envelope("error_envelope")?,
attempts: u32::try_from(row.try_get::<i32, _>("attempts").map_err(unavailable_row)?)
.unwrap_or_default(),
version: u32::try_from(row.try_get::<i32, _>("version").map_err(unavailable_row)?)
.unwrap_or_default(),
created_at_unix_ms: row.try_get("created_ms").map_err(unavailable_row)?,
updated_at_unix_ms: row.try_get("updated_ms").map_err(unavailable_row)?,
})
}
fn unavailable_row(error: sqlx::Error) -> TaskError {
PostgresTaskStore::unavailable(error)
}
@@ -0,0 +1,196 @@
//! Acceptance half of the PostgreSQL Task store (Story 4.2/4.3): the one-transaction acceptance
//! path, its bounded unique-conflict reread and the typed miss classification.
use crank_core::{
AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence, NoTasksFallback,
PlatformApiKeyId, TaskAcceptRequest, TaskAcceptance, TaskError, TaskFinishRequest, TaskState,
};
use sqlx::{Postgres, Row, Transaction};
use super::super::admission::{
AttemptError, bounded_queue_len, cleanup_expired_unlinked, insert_admission_entry,
lock_admission_in_tx,
};
use super::{
Accepted, PostgresTaskStore, TASK_PROJECTION, acceptance_from_record, policy_mode_str,
record_from_row, task_sql_error,
};
impl PostgresTaskStore {
pub(super) async fn accept_locked(
&self,
transaction: &mut Transaction<'_, Postgres>,
request: &TaskAcceptRequest,
) -> Result<Accepted, AttemptError> {
// Replay first: an already accepted Task is returned without capacity mutation, so it works
// with a full queue and after the original execution deadline while the record is retained.
if let Some((acceptance, fingerprint)) =
self.stored_acceptance(transaction, request).await?
{
return Ok(if fingerprint == request.input_fingerprint {
Accepted::Replayed(acceptance)
} else {
Accepted::Conflict
});
}
// Serialize with synchronous admission and any concurrent acceptance of the same scope.
lock_admission_in_tx(transaction, &request.workspace_id).await?;
// The competitor may have committed while this transaction waited for the controller lock.
if let Some((acceptance, fingerprint)) =
self.stored_acceptance(transaction, request).await?
{
return Ok(if fingerprint == request.input_fingerprint {
Accepted::Replayed(acceptance)
} else {
Accepted::Conflict
});
}
cleanup_expired_unlinked(transaction).await?;
let queued = bounded_queue_len(transaction).await?;
if queued >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) {
return Err(AttemptError::Overloaded);
}
let created_ms: i64 = sqlx::query(
"insert into tasks (
task_id, workspace_id, owner_kind, owner_id, authorization_scope, operation_id,
operation_version, scoped_key, input_fingerprint, effective_input_fingerprint,
request_id, trace_id, traceparent,
trace_sampled, policy_mode, approval_reference, state, input_envelope,
execution_deadline, retain_until, executable_input_ciphertext, executable_input_key_version,
executable_input_master_key_epoch, platform_api_key_id, max_attempts, retry_backoff_step_ms
) values (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $9, $10, $11, $12, $13, $14, $15, 'queued', $16,
to_timestamp($17::double precision / 1000.0),
to_timestamp($18::double precision / 1000.0), $19, $20, $21, $22, $23, $24
)
returning (extract(epoch from created_at) * 1000)::bigint as created_ms",
)
.bind(request.task_id.as_uuid())
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(i32::try_from(request.pins.operation_version).unwrap_or(i32::MAX))
.bind(&request.scoped_key)
.bind(&request.input_fingerprint)
.bind(request.support.request_id.as_str())
.bind(request.support.trace_id.as_str())
.bind(&request.support.traceparent)
.bind(request.support.trace_sampled)
.bind(policy_mode_str(request.pins.policy_mode))
.bind(request.pins.approval_reference.as_deref())
.bind(request.input.value().clone())
.bind(request.pins.execution_deadline_unix_ms)
.bind(request.pins.retain_until_unix_ms)
.bind(request.executable_input_ciphertext.as_deref())
.bind(request.executable_input_key_version.as_deref())
.bind(request.executable_input_master_key_epoch)
.bind(
request
.platform_api_key_id
.as_ref()
.map(PlatformApiKeyId::as_str),
)
.bind(i32::try_from(request.pins.retry.max_attempts).unwrap_or(1))
.bind(request.pins.retry.backoff_step_ms)
.fetch_one(&mut **transaction)
.await
.map_err(task_sql_error)?
.try_get("created_ms")
.map_err(AttemptError::from_sql)?;
let admission_request = AdmissionRequest {
scope: AdmissionScope {
workspace_id: request.workspace_id.clone(),
agent_key: request.owner_id.clone(),
operation_id: request.operation_id.clone(),
operation_version: request.pins.operation_version,
},
mode: AdmissionMode::Background,
policy_mode: request.pins.policy_mode,
no_tasks_fallback: NoTasksFallback::Unsupported,
// Never reached: the intent row is inserted by this transaction, not by `admit`.
approval: ApprovalEvidence::NotRequired,
fingerprint: request.input_fingerprint.clone(),
expires_at_unix_ms: request.pins.execution_deadline_unix_ms,
};
let owner_token = format!("task_{}", request.task_id);
insert_admission_entry(
transaction,
&admission_request,
&owner_token,
"reserved",
Some(request.task_id.as_uuid()),
)
.await?;
Ok(Accepted::Created(TaskAcceptance {
task_id: request.task_id,
state: TaskState::Queued,
operation_version: request.pins.operation_version,
support: request.support.clone(),
created_at_unix_ms: created_ms,
retain_until_unix_ms: request.pins.retain_until_unix_ms,
replayed: false,
}))
}
/// Bounded scoped reread after a foreign unique conflict aborted the transaction.
pub(super) async fn reread_after_unique(
&self,
request: &TaskAcceptRequest,
) -> Result<TaskAcceptance, TaskError> {
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
)))
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(&request.scoped_key)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::unavailable(
"durable task acceptance ended with an unresolved unique conflict",
));
};
let record = record_from_row(&row)?;
if record.input_fingerprint != request.input_fingerprint {
return Err(TaskError::FingerprintConflict);
}
Ok(acceptance_from_record(&record))
}
/// A rejected conditional update is either a lost CAS race or a foreign/absent Task. Both are
/// reported without payload, and a foreign Task is indistinguishable from an absent one.
pub(super) async fn classify_missed_finish(&self, request: &TaskFinishRequest) -> TaskError {
let access = &request.access;
match sqlx::query(
"select state, version from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5",
)
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
{
Ok(Some(_)) => TaskError::StateConflict,
Ok(None) => TaskError::NotFound,
Err(error) => Self::unavailable(error),
}
}
}
@@ -0,0 +1,585 @@
//! Durable per-outbound attempt persistence of a Task (Story 4.4): the dispatch checkpoint, the
//! single-winner attempt resolution and the owner-scoped cancellation live here so the acceptance
//! and retention half of the store stays readable.
use crank_core::{
AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase,
AttemptResolution, BeginAttempt, TaskAccess, TaskAttempt, TaskCancelOutcome, TaskCancelRequest,
TaskEnvelope, TaskError, TaskId, TaskRecord, TaskState,
};
use sqlx::Row;
use uuid::Uuid;
use super::{PostgresTaskStore, TASK_PROJECTION, record_from_row, unavailable_row};
const ATTEMPT_PROJECTION: &str = "attempt_id, task_id, ordinal, owner, fence, input_fingerprint,
upstream_key_digest, retry_allowed,
(extract(epoch from checkpoint_at) * 1000)::bigint as checkpoint_at_ms,
(extract(epoch from resolved_at) * 1000)::bigint as resolved_at_ms,
outcome, certainty, phase";
/// Bounded attempt-resolution helper: `retry` may only be armed when the attempt itself is marked
/// safe-to-retry, the pinned budget still has room and the accepted absolute deadline still does.
impl PostgresTaskStore {
fn retry_delay(
&self,
record: &TaskRecord,
lease: &AttemptLease,
safe: bool,
now: i64,
) -> Option<i64> {
let delay = record.pins.retry.backoff_before(lease.attempt.ordinal + 1);
(safe
&& lease.attempt.retry_allowed
&& lease.attempt.ordinal < record.pins.retry.max_attempts
&& now.saturating_add(delay) < record.pins.execution_deadline_unix_ms)
.then_some(delay)
}
}
impl PostgresTaskStore {
/// Authoritative word on whether an earlier resolved attempt really reached upstream despite
/// the current one not having dispatched. Returns Some(phase) only when the latest resolved
/// attempt that actually dispatched (phase = 'dispatched') is still of unknown certainty.
pub(super) async fn unproven_dispatch_phase(
&self,
task_id: TaskId,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<Option<String>, TaskError> {
let row = sqlx::query(
"select certainty, phase from task_attempts where task_id = $1 and resolved_at is not null and phase = 'dispatched' order by ordinal desc limit 1",
)
.bind(task_id.as_uuid())
.fetch_optional(&mut **transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Ok(None);
};
let certainty: Option<String> = row.try_get("certainty").map_err(Self::unavailable)?;
if certainty.as_deref() != Some(AttemptCertainty::as_str(AttemptCertainty::Unknown)) {
return Ok(None);
}
let phase: Option<String> = row.try_get("phase").map_err(Self::unavailable)?;
Ok(Some(phase.unwrap_or_else(|| "dispatched".to_owned())))
}
}
fn attempt_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskAttempt, TaskError> {
let attempt_id: String = row.try_get("attempt_id").map_err(unavailable_row)?;
let task_id: Uuid = row.try_get("task_id").map_err(unavailable_row)?;
let outcome: Option<String> = row.try_get("outcome").map_err(unavailable_row)?;
let certainty: Option<String> = row.try_get("certainty").map_err(unavailable_row)?;
let phase: Option<String> = row.try_get("phase").map_err(unavailable_row)?;
Ok(TaskAttempt {
attempt_id,
task_id: TaskId::parse(&task_id.to_string())?,
ordinal: u32::try_from(row.try_get::<i32, _>("ordinal").map_err(unavailable_row)?)
.unwrap_or_default(),
owner: row.try_get("owner").map_err(unavailable_row)?,
fence: u64::try_from(row.try_get::<i64, _>("fence").map_err(unavailable_row)?)
.unwrap_or_default(),
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
upstream_key_digest: row
.try_get("upstream_key_digest")
.map_err(unavailable_row)?,
retry_allowed: row.try_get("retry_allowed").map_err(unavailable_row)?,
checkpoint_at_unix_ms: row.try_get("checkpoint_at_ms").map_err(unavailable_row)?,
resolved_at_unix_ms: row.try_get("resolved_at_ms").map_err(unavailable_row)?,
outcome: outcome
.as_deref()
.map(AttemptOutcomeClass::parse)
.transpose()?,
certainty: certainty
.as_deref()
.map(AttemptCertainty::parse)
.transpose()?,
phase: phase.as_deref().map(AttemptPhase::parse).transpose()?,
})
}
impl PostgresTaskStore {
pub(super) async fn checkpoint_dispatch_attempt(
&self,
request: BeginAttempt,
) -> Result<AttemptLease, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
// One row lock serializes this checkpoint with claim, cancellation and the resolution of the
// same Task, so a cancel that wins first can never be followed by an outbound call.
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks where task_id = $1 for update"
)))
.bind(request.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row)?;
// The checkpoint must bind the payload this attempt actually executes: after a committed
// input continuation that is the effective fingerprint, not the unchanged acceptance one.
if record.effective_input_fingerprint != request.input_fingerprint {
transaction.rollback().await.ok();
return Err(TaskError::FingerprintConflict);
}
let now = crank_core::current_unix_ms();
let eligible = record.state == TaskState::Working
&& record.lease_owner.as_deref() == Some(request.owner.as_str())
&& record.fence == request.fence
&& record.version == request.expected_version
&& record.current_attempt_id.is_none()
&& record.cancel_requested_at_unix_ms.is_none()
&& record.lease_until_unix_ms.is_some_and(|until| until > now)
&& record.attempts < record.pins.retry.max_attempts
&& record.pins.execution_deadline_unix_ms > now
&& record.next_attempt_at_unix_ms.is_none_or(|at| at <= now);
if !eligible {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let ordinal = record.attempts.saturating_add(1);
sqlx::query(
"insert into task_attempts (
attempt_id, task_id, ordinal, owner, fence, input_fingerprint,
upstream_key_digest, retry_allowed
) values ($1, $2, $3, $4, $5, $6, $7, $8)",
)
.bind(&request.attempt_id)
.bind(request.task_id.as_uuid())
.bind(i32::try_from(ordinal).unwrap_or(i32::MAX))
.bind(&request.owner)
.bind(i64::try_from(request.fence).unwrap_or_default())
.bind(&request.input_fingerprint)
.bind(request.upstream_key_digest.as_deref())
.bind(request.retry_allowed)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set current_attempt_id = $1,
dispatch_started_at = now(),
attempts = attempts + 1,
next_attempt_at = null,
version = version + 1,
updated_at = now()
where task_id = $2 and current_attempt_id is null and version = $3
returning {TASK_PROJECTION}"
)))
.bind(&request.attempt_id)
.bind(request.task_id.as_uuid())
.bind(i64::from(record.version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let task = record_from_row(&row)?;
transaction.commit().await.map_err(Self::unavailable)?;
let attempt = TaskAttempt {
attempt_id: request.attempt_id,
task_id: request.task_id,
ordinal,
owner: request.owner,
fence: request.fence,
input_fingerprint: request.input_fingerprint,
upstream_key_digest: request.upstream_key_digest,
retry_allowed: request.retry_allowed,
checkpoint_at_unix_ms: task.dispatch_started_at_unix_ms.unwrap_or(now),
resolved_at_unix_ms: None,
outcome: None,
certainty: None,
phase: None,
};
Ok(AttemptLease { attempt, task })
}
pub(super) async fn resolve_attempt_outcome(
&self,
lease: &AttemptLease,
outcome: AttemptOutcome,
) -> Result<AttemptResolution, TaskError> {
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks where task_id = $1 for update"
)))
.bind(lease.attempt.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row)?;
let now = crank_core::current_unix_ms();
let prior_uncertain_phase = self
.unproven_dispatch_phase(lease.attempt.task_id, &mut transaction)
.await?;
// Only the current attempt of the current owner, under the version it was admitted with, may
// be resolved. A stale worker can never append a second outcome to the history or the Task.
let current = record.state == TaskState::Working
&& record.lease_owner.as_deref() == Some(lease.attempt.owner.as_str())
&& record.fence == lease.attempt.fence
&& record.version == lease.task.version
&& record.current_attempt_id.as_deref() == Some(lease.attempt.attempt_id.as_str())
&& record.lease_until_unix_ms.is_some_and(|until| until > now);
if !current {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let (class, certainty, phase) = match &outcome {
AttemptOutcome::Completed(_) => (
AttemptOutcomeClass::Completed,
AttemptCertainty::Known,
AttemptPhase::Dispatched,
),
AttemptOutcome::Failed { phase, .. } => {
(AttemptOutcomeClass::Failed, AttemptCertainty::Known, *phase)
}
AttemptOutcome::Uncertain { phase, .. } => (
AttemptOutcomeClass::OutcomeUnknown,
AttemptCertainty::Unknown,
*phase,
),
};
// A failed or uncertain attempt that may be safely repeated keeps the lease and arms the
// pinned backoff. Exhausted budget or deadline never turns uncertainty into a plain failure.
let retry = match &outcome {
AttemptOutcome::Failed {
retryable,
phase: _,
error: _,
} => self.retry_delay(&record, lease, *retryable, now),
AttemptOutcome::Uncertain {
repeatable,
phase: _,
error: _,
} => self.retry_delay(&record, lease, *repeatable, now),
AttemptOutcome::Completed(_) => None,
};
let attempt_class = if retry.is_some() {
AttemptOutcomeClass::RetryScheduled
} else {
class
};
let closed = sqlx::query(
"update task_attempts
set resolved_at = now(), outcome = $1, certainty = $2, phase = $3
where attempt_id = $4 and resolved_at is null",
)
.bind(attempt_class.as_str())
.bind(certainty.as_str())
.bind(phase.as_str())
.bind(&lease.attempt.attempt_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
if closed.rows_affected() != 1 {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let (state, result, error, keep_lease, next_attempt_at) = match outcome {
AttemptOutcome::Completed(result) => {
(TaskState::Completed, Some(result), None, false, None)
}
AttemptOutcome::Failed {
error,
retryable: _,
phase,
} => match retry {
Some(delay) => (
TaskState::Working,
None,
None,
true,
Some(now.saturating_add(delay)),
),
None => {
if phase == AttemptPhase::BeforeDispatch {
if let Some(prior) = prior_uncertain_phase.as_ref() {
let mut value = error.value().clone();
if let serde_json::Value::Object(map) = &mut value {
map.insert(
"phase".to_string(),
serde_json::Value::String(prior.clone()),
);
}
let replaced = TaskEnvelope::try_redacted(&value)?;
(TaskState::OutcomeUnknown, None, Some(replaced), false, None)
} else {
(TaskState::Failed, None, Some(error), false, None)
}
} else {
(TaskState::Failed, None, Some(error), false, None)
}
}
},
AttemptOutcome::Uncertain {
error,
repeatable: _,
phase: _,
} => match retry {
Some(delay) => (
TaskState::Working,
None,
None,
true,
Some(now.saturating_add(delay)),
),
None => (TaskState::OutcomeUnknown, None, Some(error), false, None),
},
};
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = $1,
result_envelope = $2,
error_envelope = $3,
lease_owner = case when $4 then lease_owner else null end,
lease_until = case when $4 then lease_until else null end,
-- A retry-scheduled attempt is already resolved, so it no longer holds the Task:
-- the next durable attempt needs its own checkpoint under the same lease.
current_attempt_id = case when $8 then null else current_attempt_id end,
next_attempt_at = case when $5::bigint is null then null
else to_timestamp($5::double precision / 1000.0) end,
version = version + 1,
updated_at = now()
where task_id = $6 and version = $7
returning {TASK_PROJECTION}"
)))
.bind(state.as_str())
.bind(result.as_ref().map(|value| value.value().clone()))
.bind(error.as_ref().map(|value| value.value().clone()))
.bind(keep_lease)
.bind(next_attempt_at)
.bind(lease.attempt.task_id.as_uuid())
.bind(i64::from(record.version))
.bind(retry.is_some())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let task = record_from_row(&row)?;
if !keep_lease {
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(lease.attempt.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
}
transaction.commit().await.map_err(Self::unavailable)?;
Ok(match retry {
Some(_) => AttemptResolution::RetryAt {
next_attempt_at_unix_ms: next_attempt_at.unwrap_or(now),
task,
},
None if state == TaskState::OutcomeUnknown => AttemptResolution::Unknown(task),
None => AttemptResolution::Finished(task),
})
}
pub(super) async fn cancel_task(
&self,
request: TaskCancelRequest,
) -> Result<TaskCancelOutcome, TaskError> {
request.validate()?;
let access = &request.access;
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5
for update"
)))
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row)?;
if let Some(expected) = request.expected_version
&& record.version != expected
{
transaction.rollback().await.ok();
return Ok(TaskCancelOutcome::Conflict(record));
}
if record.state.is_terminal() {
transaction.rollback().await.ok();
return Ok(TaskCancelOutcome::AlreadyTerminal(record));
}
if record.state == TaskState::OutcomeUnknown {
transaction.rollback().await.ok();
return Ok(TaskCancelOutcome::OutcomeUnknown(record));
}
// A checkpointed attempt may already have reached the upstream: the honest answer is the
// non-terminal uncertain state, never a plain cancellation and never a retry.
let checkpointed = record.current_attempt_id.clone();
// Preserve an uncertainty recorded by a previously resolved attempt even after RetryAt
// cleared current_attempt_id: the request already reached the upstream in that case.
let prior_uncertain = self
.unproven_dispatch_phase(request.task_id, &mut transaction)
.await?
.is_some();
let (state, code) = match (&checkpointed, prior_uncertain) {
(Some(_), _) => (TaskState::OutcomeUnknown, "cancelled_after_dispatch"),
(None, true) => (TaskState::OutcomeUnknown, "cancelled_after_dispatch"),
(None, false) => (TaskState::Cancelled, "cancelled_before_dispatch"),
};
if let Some(attempt_id) = checkpointed {
let closed = sqlx::query(
"update task_attempts
set resolved_at = now(), outcome = 'outcome_unknown', certainty = 'unknown',
phase = 'dispatched'
where task_id = $1 and attempt_id = $2 and resolved_at is null",
)
.bind(request.task_id.as_uuid())
.bind(&attempt_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let _ = closed;
}
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = $1,
error_envelope = jsonb_build_object('code', $2),
cancel_requested_at = now(),
cancel_reason = $3,
lease_owner = null,
lease_until = null,
version = version + 1,
updated_at = now()
where task_id = $4 and version = $5
returning {TASK_PROJECTION}"
)))
.bind(state.as_str())
.bind(code)
.bind(&request.reason)
.bind(request.task_id.as_uuid())
.bind(i64::from(record.version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let task = record_from_row(&row)?;
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(request.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(match state {
TaskState::Cancelled => TaskCancelOutcome::Cancelled(task),
_ => TaskCancelOutcome::OutcomeUnknown(task),
})
}
pub(super) async fn refuse_attempt_resolution(
&self,
lease: &crank_core::TaskLease,
code: &str,
) -> Result<TaskRecord, TaskError> {
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks\n where task_id = $1\n for update"
)))
.bind(lease.record.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
let now = crank_core::current_unix_ms();
let eligible = matches!(record.state, TaskState::Queued | TaskState::Working)
&& record.lease_owner.as_deref() == Some(lease.owner.as_str())
&& record.fence == lease.fence
&& record.version == lease.record.version
&& record.current_attempt_id.is_none()
&& record.lease_until_unix_ms.is_some_and(|until| until > now);
if !eligible {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let prior_phase = self
.unproven_dispatch_phase(lease.record.task_id, &mut transaction)
.await?;
let uncertain = prior_phase.is_some();
let state = if uncertain {
TaskState::OutcomeUnknown
} else {
TaskState::Failed
};
let phase = prior_phase.unwrap_or_else(|| "before_dispatch".to_owned());
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks\n set state = $1,\n error_envelope = jsonb_build_object('code', $2, 'phase', $3),\n lease_owner = null,\n lease_until = null,\n next_attempt_at = null,\n version = version + 1,\n updated_at = now()\n where task_id = $4 and version = $5\n returning {TASK_PROJECTION}"
)))
.bind(state.as_str())
.bind(code)
.bind(&phase)
.bind(lease.record.task_id.as_uuid())
.bind(i64::from(record.version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let task = record_from_row(&row).map_err(Self::unavailable)?;
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(lease.record.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(task)
}
pub(super) async fn attempt_history(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<Vec<TaskAttempt>, TaskError> {
access.validate()?;
let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {ATTEMPT_PROJECTION} from task_attempts
where task_id = $1 and exists (
select 1 from tasks
where tasks.task_id = task_attempts.task_id and tasks.workspace_id = $2
and tasks.owner_kind = $3 and tasks.owner_id = $4
and tasks.authorization_scope = $5
)
order by ordinal"
)))
.bind(task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_all(&self.pool)
.await
.map_err(Self::unavailable)?;
rows.iter().map(attempt_from_row).collect()
}
}
@@ -0,0 +1,659 @@
//! Input-continuation half of the PostgreSQL Task store (Story 4.5, FR-25).
//!
//! One transaction owns each transition: suspend (`working` -> `input_required`), the committed
//! answer (`input_required` -> `working`) and the TTL expiry (`input_required` -> `failed`). Authority,
//! generation, TTL and receipt checks run under the same Task row lock, so a revocation, a competing
//! answer, a sweeper or a cancellation has exactly one winner and never a duplicated side effect.
//!
//! Raw input never enters a request, receipt, audit row or error text: the validated payload is
//! encrypted by the runtime service before this module sees it.
use crank_core::{
TaskAccess, TaskError, TaskInputRequest, TaskInputStore, TaskInputUpdateOutcome,
TaskInputUpdateRequest, TaskRecord, TaskState,
};
use sqlx::{Postgres, Row, Transaction};
use super::{PostgresTaskStore, TASK_PROJECTION, record_from_row};
#[async_trait::async_trait]
impl TaskInputStore for PostgresTaskStore {
async fn suspend_for_input(
&self,
lease: &crank_core::TaskLease,
request: TaskInputRequest,
) -> Result<TaskRecord, TaskError> {
self.suspend_input_locked(lease, request).await
}
async fn input_request(
&self,
access: &TaskAccess,
task_id: crank_core::TaskId,
) -> Result<TaskInputRequest, TaskError> {
self.read_input_request(access, task_id).await
}
async fn read_executable_input_for_update(
&self,
access: &TaskAccess,
task_id: crank_core::TaskId,
) -> Result<(String, String, i64), TaskError> {
self.read_binding_for_update(access, task_id).await
}
async fn apply_input(
&self,
request: TaskInputUpdateRequest,
) -> Result<TaskInputUpdateOutcome, TaskError> {
self.apply_input_locked(request).await
}
async fn expire_input_waits(&self, batch: u32) -> Result<u64, TaskError> {
self.expire_input_waits_locked(batch).await
}
}
impl PostgresTaskStore {
/// One transactional suspend of a leased Task before any dispatch checkpoint.
pub(super) async fn suspend_input_locked(
&self,
lease: &crank_core::TaskLease,
request: TaskInputRequest,
) -> Result<TaskRecord, TaskError> {
request.validate()?;
if request.expected_version != lease.record.version.saturating_add(1) {
return Err(TaskError::StateConflict);
}
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks where task_id = $1 for update"
)))
.bind(lease.record.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row)?;
let current = record.state == TaskState::Working
&& record.lease_owner.as_deref() == Some(lease.owner.as_str())
&& record.fence == lease.fence
&& record.version == lease.record.version
&& record.current_attempt_id.is_none()
&& record.attempts == 0
&& record.cancel_requested_at_unix_ms.is_none()
&& record
.lease_until_unix_ms
.is_some_and(|until| until > crank_core::current_unix_ms())
&& record.pins.execution_deadline_unix_ms > crank_core::current_unix_ms();
// A generation that already dispatched may not be suspended: that would hide a possible side
// effect behind an apparent "waiting for input".
if !current {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
if request.expires_at_unix_ms > record.pins.execution_deadline_unix_ms {
transaction.rollback().await.ok();
return Err(TaskError::invalid(
"task input TTL exceeds the execution deadline",
));
}
sqlx::query(
"insert into task_input_requests (
task_id, wait_id, expected_version, schema_revision, schema_format_version,
fingerprint_format_version, schema_snapshot, projected_schema, prompt_code,
missing_fields, expires_at
) values (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
to_timestamp($11::double precision / 1000.0)
)",
)
.bind(lease.record.task_id.as_uuid())
.bind(&request.wait_id)
.bind(i32::try_from(request.expected_version).unwrap_or(i32::MAX))
.bind(i32::try_from(request.schema_revision).unwrap_or(i32::MAX))
.bind(i32::try_from(request.schema_format_version).unwrap_or(i32::MAX))
.bind(i32::try_from(request.fingerprint_format_version).unwrap_or(i32::MAX))
.bind(request.schema_snapshot.clone())
.bind(request.projected_schema.clone())
.bind(&request.prompt_code)
.bind(serde_json::Value::Array(
request
.missing_fields
.iter()
.cloned()
.map(serde_json::Value::String)
.collect(),
))
.bind(request.expires_at_unix_ms)
.execute(&mut *transaction)
.await
.map_err(|error| {
match error
.as_database_error()
.and_then(|db| db.code())
.as_deref()
{
Some("23505") => TaskError::StateConflict,
_ => Self::unavailable(error),
}
})?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = 'input_required',
lease_owner = null,
lease_until = null,
version = version + 1,
updated_at = now()
where task_id = $1 and state = 'working' and version = $2
and lease_owner = $3 and fence = $4 and current_attempt_id is null
and lease_until > clock_timestamp()
returning {TASK_PROJECTION}"
)))
.bind(lease.record.task_id.as_uuid())
.bind(i64::from(record.version))
.bind(&lease.owner)
.bind(i64::try_from(lease.fence).unwrap_or_default())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let suspended = record_from_row(&row)?;
insert_audit(
&mut transaction,
lease.record.task_id,
&request.wait_id,
"input_required",
TaskState::Working,
TaskState::InputRequired,
record.version,
suspended.version,
None,
None,
)
.await?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(suspended)
}
/// Owner-scoped read of the durable request; a foreign Task is one opaque not-found.
pub(super) async fn read_input_request(
&self,
access: &TaskAccess,
task_id: crank_core::TaskId,
) -> Result<TaskInputRequest, TaskError> {
access.validate()?;
let row = sqlx::query(
"select r.wait_id, r.expected_version, r.schema_revision, r.schema_format_version,
r.fingerprint_format_version, r.schema_snapshot, r.projected_schema,
r.prompt_code, r.missing_fields,
(extract(epoch from r.expires_at) * 1000)::bigint as expires_at_ms
from task_input_requests r
join tasks t on t.task_id = r.task_id
where r.task_id = $1 and t.workspace_id = $2 and t.owner_kind = $3
and t.owner_id = $4 and t.authorization_scope = $5",
)
.bind(task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::NotFound);
};
let missing_fields: Vec<String> = serde_json::from_value(
row.try_get::<serde_json::Value, _>("missing_fields")
.map_err(Self::unavailable)?,
)
.map_err(|_| TaskError::unavailable("stored input request metadata is invalid"))?;
Ok(TaskInputRequest {
wait_id: row.try_get("wait_id").map_err(Self::unavailable)?,
expected_version: u32::try_from(
row.try_get::<i32, _>("expected_version")
.map_err(Self::unavailable)?,
)
.unwrap_or_default(),
schema_revision: u32::try_from(
row.try_get::<i32, _>("schema_revision")
.map_err(Self::unavailable)?,
)
.unwrap_or_default(),
schema_format_version: u32::try_from(
row.try_get::<i32, _>("schema_format_version")
.map_err(Self::unavailable)?,
)
.unwrap_or_default(),
schema_snapshot: row.try_get("schema_snapshot").map_err(Self::unavailable)?,
projected_schema: row.try_get("projected_schema").map_err(Self::unavailable)?,
prompt_code: row.try_get("prompt_code").map_err(Self::unavailable)?,
missing_fields,
expires_at_unix_ms: row.try_get("expires_at_ms").map_err(Self::unavailable)?,
fingerprint_format_version: u32::try_from(
row.try_get::<i32, _>("fingerprint_format_version")
.map_err(Self::unavailable)?,
)
.unwrap_or_default(),
})
}
/// Owner-scoped read of the binding a continuation may extend. Never lease-authorized.
///
/// Deliberately not filtered by lifecycle state: an identical replay of an already committed
/// answer (including after completion or TTL expiry) must reach the receipt lookup, which is the
/// single authority for that outcome.
pub(super) async fn read_binding_for_update(
&self,
access: &TaskAccess,
task_id: crank_core::TaskId,
) -> Result<(String, String, i64), TaskError> {
access.validate()?;
let row = sqlx::query(
"select executable_input_ciphertext, executable_input_key_version,
executable_input_master_key_epoch
from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5",
)
.bind(task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::NotFound);
};
let ciphertext: Option<String> = row
.try_get("executable_input_ciphertext")
.map_err(Self::unavailable)?;
let key_version: Option<String> = row
.try_get("executable_input_key_version")
.map_err(Self::unavailable)?;
let epoch: Option<i64> = row
.try_get("executable_input_master_key_epoch")
.map_err(Self::unavailable)?;
match (ciphertext, key_version, epoch) {
(Some(ciphertext), Some(key_version), Some(epoch)) if !ciphertext.is_empty() => {
Ok((ciphertext, key_version, epoch))
}
_ => Err(TaskError::NotFound),
}
}
/// One transactional answer application: authority, receipt, generation, TTL and CAS.
pub(super) async fn apply_input_locked(
&self,
request: TaskInputUpdateRequest,
) -> Result<TaskInputUpdateOutcome, TaskError> {
request.validate()?;
let access = &request.access;
let mut transaction = self.start().await?;
super::super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
.await
.map_err(|error| match error {
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
_ => TaskError::StateConflict,
})?;
super::super::master_key::ensure_active_master_key_epoch(
&mut transaction,
request.executable_input_master_key_epoch,
)
.await
.map_err(|error| match error {
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
_ => TaskError::StateConflict,
})?;
// Authority is revalidated inside this transaction, before any receipt lookup and under the
// same lock order: master key check, platform key share lock, Task row lock.
let key_row = sqlx::query(
"select k.id from platform_api_keys k
join tasks t on t.platform_api_key_id = k.id
where t.task_id = $1 and t.owner_kind = 'agent'
and k.workspace_id = $2 and k.agent_id = $3
and k.key_kind = 'mcp_client' and k.status = 'active'
and (k.expires_at is null or k.expires_at > clock_timestamp())
and (k.scopes_json ? 'write' or k.scopes_json ? 'deploy')
for share of k",
)
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(&access.owner_id)
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
if key_row.is_none() {
transaction.rollback().await.ok();
// A revoked key, a foreign scope and an absent Task share one opaque answer.
return Err(TaskError::NotFound);
}
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5
for update"
)))
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row)?;
// Receipt lookup precedes the generation and TTL checks: an authorized identical replay keeps
// returning the committed outcome even after completion, failure or TTL expiry.
let receipt = sqlx::query(
"select wait_id, update_key_digest, canonical_fingerprint, committed_state,
committed_version
from task_input_requests where task_id = $1 and accepted_at is not null",
)
.bind(request.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
if let Some(receipt) = receipt {
let wait_id: String = receipt.try_get("wait_id").map_err(Self::unavailable)?;
if wait_id != request.wait_id {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let stored_key: String = receipt
.try_get("update_key_digest")
.map_err(Self::unavailable)?;
if stored_key != request.update_key_digest {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let stored_fingerprint: String = receipt
.try_get("canonical_fingerprint")
.map_err(Self::unavailable)?;
if stored_fingerprint != request.canonical_fingerprint {
transaction.rollback().await.ok();
return Err(TaskError::FingerprintConflict);
}
let state: String = receipt
.try_get("committed_state")
.map_err(Self::unavailable)?;
let version: i32 = receipt
.try_get("committed_version")
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
return Ok(TaskInputUpdateOutcome {
task_id: request.task_id,
wait_id,
state: TaskState::parse(&state)?,
version: u32::try_from(version).unwrap_or_default(),
replayed: true,
});
}
let pending = sqlx::query(
"select wait_id, expected_version,
(extract(epoch from expires_at) * 1000)::bigint as expires_at_ms,
(extract(epoch from clock_timestamp()) * 1000)::bigint as now_ms
from task_input_requests where task_id = $1 and accepted_at is null",
)
.bind(request.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(pending) = pending else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let wait_id: String = pending.try_get("wait_id").map_err(Self::unavailable)?;
let expected_version: i32 = pending
.try_get("expected_version")
.map_err(Self::unavailable)?;
let expires_at_ms: i64 = pending
.try_get("expires_at_ms")
.map_err(Self::unavailable)?;
// PostgreSQL clock, read after the row lock was taken: never a client timestamp.
let now_ms: i64 = pending.try_get("now_ms").map_err(Self::unavailable)?;
if wait_id != request.wait_id
|| u32::try_from(expected_version).unwrap_or_default() != request.expected_version
|| record.version != request.expected_version
|| record.state != TaskState::InputRequired
{
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
if record.cancel_requested_at_unix_ms.is_some() {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
if now_ms >= expires_at_ms {
// The same transactional helper the sweeper uses: the Task ends failed with an audit
// reason and no dispatch. The update never revives it.
expire_locked(&mut transaction, &record, &wait_id, now_ms).await?;
transaction.commit().await.map_err(Self::unavailable)?;
return Err(TaskError::InputExpired);
}
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = 'working',
effective_input_fingerprint = $1,
executable_input_ciphertext = $2,
executable_input_key_version = $3,
executable_input_master_key_epoch = $4,
lease_owner = null,
lease_until = null,
version = version + 1,
updated_at = now()
where task_id = $5 and state = 'input_required' and version = $6
returning {TASK_PROJECTION}"
)))
.bind(&request.effective_input_fingerprint)
.bind(&request.executable_input_ciphertext)
.bind(&request.executable_input_key_version)
.bind(request.executable_input_master_key_epoch)
.bind(request.task_id.as_uuid())
.bind(i64::from(record.version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let accepted = record_from_row(&row)?;
let updated = sqlx::query(
"update task_input_requests
set accepted_at = now(),
update_key_digest = $1,
canonical_fingerprint = $2,
effective_input_fingerprint = $3,
committed_state = 'working',
committed_version = $4
where task_id = $5 and wait_id = $6 and accepted_at is null",
)
.bind(&request.update_key_digest)
.bind(&request.canonical_fingerprint)
.bind(&request.effective_input_fingerprint)
.bind(i64::from(accepted.version))
.bind(request.task_id.as_uuid())
.bind(&request.wait_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
if updated.rows_affected() != 1 {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
insert_audit(
&mut transaction,
request.task_id,
&request.wait_id,
"input_accepted",
TaskState::InputRequired,
TaskState::Working,
record.version,
accepted.version,
Some(access.owner_id.as_str()),
None,
)
.await?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(TaskInputUpdateOutcome {
task_id: request.task_id,
wait_id: request.wait_id,
state: TaskState::Working,
version: accepted.version,
replayed: false,
})
}
/// Bounded TTL sweep, safe to call on every poll while all worker slots are busy.
pub(super) async fn expire_input_waits_locked(&self, batch: u32) -> Result<u64, TaskError> {
let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT));
let mut transaction = self.start().await?;
let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks t
where t.state = 'input_required'
and exists (
select 1 from task_input_requests r
where r.task_id = t.task_id and r.accepted_at is null
and r.expires_at <= clock_timestamp()
)
order by t.task_id
limit $1
for update of t skip locked"
)))
.bind(batch)
.fetch_all(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let mut expired = 0u64;
for row in rows {
let record = record_from_row(&row)?;
let Some(wait_id) = sqlx::query_scalar::<_, String>(
"select wait_id from task_input_requests
where task_id = $1 and accepted_at is null and expires_at <= clock_timestamp()",
)
.bind(record.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?
else {
continue;
};
if expire_locked(
&mut transaction,
&record,
&wait_id,
crank_core::current_unix_ms(),
)
.await?
{
expired += 1;
}
}
transaction.commit().await.map_err(Self::unavailable)?;
Ok(expired)
}
}
/// Marks one locked `input_required` Task failed, releases its admission permit and writes the audit
/// reason. Returns whether this call performed the transition.
async fn expire_locked(
transaction: &mut Transaction<'_, Postgres>,
record: &TaskRecord,
wait_id: &str,
_now_ms: i64,
) -> Result<bool, TaskError> {
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = 'failed',
error_envelope = jsonb_build_object('reason', 'input_ttl_expired'),
lease_owner = null,
lease_until = null,
version = version + 1,
updated_at = now()
where task_id = $1 and state = 'input_required' and version = $2
returning {TASK_PROJECTION}"
)))
.bind(record.task_id.as_uuid())
.bind(i64::from(record.version))
.fetch_optional(&mut **transaction)
.await
.map_err(PostgresTaskStore::unavailable)?;
let Some(row) = row else {
return Ok(false);
};
let failed = record_from_row(&row)?;
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(record.task_id.as_uuid())
.execute(&mut **transaction)
.await
.map_err(PostgresTaskStore::unavailable)?;
insert_audit(
transaction,
record.task_id,
wait_id,
"input_expired",
TaskState::InputRequired,
TaskState::Failed,
record.version,
failed.version,
None,
Some("input_ttl_expired"),
)
.await?;
Ok(true)
}
#[allow(clippy::too_many_arguments)]
async fn insert_audit(
transaction: &mut Transaction<'_, Postgres>,
task_id: crank_core::TaskId,
wait_id: &str,
event: &str,
from_state: TaskState,
to_state: TaskState,
from_version: u32,
to_version: u32,
actor: Option<&str>,
reason_code: Option<&str>,
) -> Result<(), TaskError> {
sqlx::query(
"insert into task_input_audit (
task_id, wait_id, event, from_state, to_state, from_version, to_version, actor,
reason_code
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
)
.bind(task_id.as_uuid())
.bind(wait_id)
.bind(event)
.bind(from_state.as_str())
.bind(to_state.as_str())
.bind(i32::try_from(from_version).unwrap_or(i32::MAX))
.bind(i32::try_from(to_version).unwrap_or(i32::MAX))
.bind(actor)
.bind(reason_code)
.execute(&mut **transaction)
.await
.map_err(PostgresTaskStore::unavailable)?;
Ok(())
}
@@ -0,0 +1,953 @@
//! PostgreSQL implementation of the durable Task store (Story 4.2, FR-22/FR-23).
//!
//! One PostgreSQL transaction owns the whole acceptance: Task record, dispatch intent
//! (`execution_admission_entries.task_id`) and its queue slot commit together, so no Task ID and no
//! acknowledgement can exist without the durable record. The transaction reuses the transaction-aware
//! primitives of the execution-admission store (`lock_admission_in_tx`, `bounded_queue_len`,
//! `insert_admission_entry`) instead of copying capacity SQL or starting a nested transaction.
//!
//! Replay is resolved before any capacity mutation and never touches the accepted row. An ambiguous
//! COMMIT is reported as an unavailable store: it is never treated as a proven rollback, so no
//! compensating delete and no synchronous fallback can happen after an uncertain acceptance.
use async_trait::async_trait;
use crank_core::{
AttemptLease, AttemptOutcome, AttemptResolution, BeginAttempt, PlatformApiKeyId,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome,
TaskCancelRequest, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskLease,
TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskRetryPolicy, TaskState, TaskStore,
TaskSupportIds, ToolExecutionMode,
};
use serde_json::Value;
use sqlx::{PgPool, Postgres, Row, Transaction};
use uuid::Uuid;
use super::admission::{AdmissionLimits, AttemptError};
mod acceptance;
mod attempts;
mod input;
const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_id,
authorization_scope, platform_api_key_id, operation_id, input_fingerprint,
effective_input_fingerprint, operation_version,
policy_mode,
approval_reference,
request_id, trace_id, traceparent, trace_sampled, input_envelope, progress_envelope,
result_envelope, error_envelope, attempts, fence, lease_owner, lease_until,
(extract(epoch from lease_until) * 1000)::bigint as lease_until_ms,
current_attempt_id, dispatch_started_at,
(extract(epoch from dispatch_started_at) * 1000)::bigint as dispatch_started_at_ms, version,
max_attempts, retry_backoff_step_ms,
(extract(epoch from next_attempt_at) * 1000)::bigint as next_attempt_at_ms,
(extract(epoch from cancel_requested_at) * 1000)::bigint as cancel_requested_at_ms,
cancel_reason,
(extract(epoch from execution_deadline) * 1000)::bigint as execution_deadline_ms,
(extract(epoch from retain_until) * 1000)::bigint as retain_until_ms,
(extract(epoch from created_at) * 1000)::bigint as created_ms,
(extract(epoch from updated_at) * 1000)::bigint as updated_ms";
#[derive(Clone)]
pub struct PostgresTaskStore {
pool: PgPool,
limits: AdmissionLimits,
scheduler: crank_core::AdmissionSchedulerConfig,
}
impl PostgresTaskStore {
pub fn new(
pool: PgPool,
limits: AdmissionLimits,
scheduler: crank_core::AdmissionSchedulerConfig,
) -> Result<Self, TaskError> {
limits
.validate()
.map_err(|error| TaskError::invalid(error.to_string()))?;
scheduler
.validate()
.map_err(|error| TaskError::invalid(error.to_string()))?;
Ok(Self {
pool,
limits,
scheduler,
})
}
fn unavailable(error: impl std::fmt::Display) -> TaskError {
TaskError::unavailable(format!("durable task store is unavailable: {error}"))
}
async fn start(&self) -> Result<Transaction<'_, Postgres>, TaskError> {
let mut transaction = self.pool.begin().await.map_err(Self::unavailable)?;
sqlx::query("set local lock_timeout = '3000ms'")
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
sqlx::query("set local statement_timeout = '5000ms'")
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
Ok(transaction)
}
/// Scoped lookup of an accepted Task. The full owner/authorization/Tool predicate is part of the
/// query, so a foreign row is never fetched.
async fn stored_acceptance(
&self,
transaction: &mut Transaction<'_, Postgres>,
request: &TaskAcceptRequest,
) -> Result<Option<(TaskAcceptance, String)>, AttemptError> {
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
)))
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(&request.scoped_key)
.fetch_optional(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let Some(row) = row else {
return Ok(None);
};
let record = record_from_row(&row).map_err(AttemptError::Failure)?;
Ok(Some((
acceptance_from_record(&record),
record.input_fingerprint,
)))
}
}
fn acceptance_from_record(record: &TaskRecord) -> TaskAcceptance {
TaskAcceptance {
task_id: record.task_id,
state: record.state,
operation_version: record.pins.operation_version,
support: record.support.clone(),
created_at_unix_ms: record.created_at_unix_ms,
retain_until_unix_ms: record.pins.retain_until_unix_ms,
replayed: true,
}
}
/// Readiness probe of the durable Task state: the table must be reachable and readable. A missing or
/// unreadable schema is a dependency failure, never a degraded-but-ready state.
pub async fn tasks_health(pool: &PgPool) -> Result<(), TaskError> {
sqlx::query("select count(*)::bigint from tasks where false")
.fetch_one(pool)
.await
.map_err(PostgresTaskStore::unavailable)?;
Ok(())
}
/// One transaction's decision of the acceptance path.
enum Accepted {
Created(TaskAcceptance),
Replayed(TaskAcceptance),
Conflict,
}
#[async_trait]
impl TaskStore for PostgresTaskStore {
async fn health(&self) -> Result<(), TaskError> {
sqlx::query("select count(*)::bigint from tasks where false")
.fetch_one(&self.pool)
.await
.map_err(Self::unavailable)?;
super::admission::admission_health(&self.pool)
.await
.map_err(|error| TaskError::unavailable(error.to_string()))?;
Ok(())
}
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
.await
.map_err(|error| match error {
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
_ => TaskError::StateConflict,
})?;
if let Some(epoch) = request.executable_input_master_key_epoch {
super::master_key::ensure_active_master_key_epoch(&mut transaction, epoch)
.await
.map_err(|error| match error {
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
_ => TaskError::StateConflict,
})?;
}
let outcome = self.accept_locked(&mut transaction, &request).await;
match outcome {
Ok(Accepted::Created(acceptance) | Accepted::Replayed(acceptance)) => {
// The acknowledgement of a created Task is only possible after a real COMMIT.
transaction.commit().await.map_err(Self::unavailable)?;
Ok(TaskAcceptance {
replayed: acceptance.replayed,
..acceptance
})
}
Ok(Accepted::Conflict) => {
transaction.rollback().await.ok();
Err(TaskError::FingerprintConflict)
}
Err(AttemptError::Overloaded) => {
transaction.rollback().await.ok();
Err(TaskError::Overloaded {
retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS,
})
}
Err(AttemptError::Contended) => {
transaction.rollback().await.ok();
Err(TaskError::unavailable(
"durable task acceptance could not take the admission locks",
))
}
Err(AttemptError::UniqueViolation) => {
// Roll back the aborted transaction, then reread the scoped key once outside it.
transaction.rollback().await.ok();
self.reread_after_unique(&request).await
}
Err(AttemptError::Failure(error)) => {
transaction.rollback().await.ok();
Err(error)
}
Err(AttemptError::AdmissionFailure(error)) => {
transaction.rollback().await.ok();
Err(TaskError::unavailable(error.to_string()))
}
}
}
async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result<TaskRecord, TaskError> {
access.validate()?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5"
)))
.bind(task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::NotFound);
};
record_from_row(&row).map_err(Self::unavailable)
}
async fn find(
&self,
access: &TaskAccess,
operation_id: &str,
scoped_key: &str,
) -> Result<Option<TaskRecord>, TaskError> {
access.validate()?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
)))
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.bind(operation_id)
.bind(scoped_key)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
row.map(|row| record_from_row(&row)).transpose()
}
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
let access = &request.access;
let update = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = $1,
result_envelope = $2,
error_envelope = $3,
version = version + 1,
updated_at = now()
where task_id = $4 and workspace_id = $5 and owner_kind = $6
and owner_id = $7 and authorization_scope = $8
and state = $9 and version = $10
and (lease_owner is null or fence = 0)
and current_attempt_id is null
returning {TASK_PROJECTION}"
)))
.bind(request.target_state.as_str())
.bind(request.result.as_ref().map(|value| value.value().clone()))
.bind(request.error.as_ref().map(|value| value.value().clone()))
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.bind(request.expected_state.as_str())
.bind(i64::from(request.expected_version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = update else {
transaction.rollback().await.ok();
return Err(self.classify_missed_finish(&request).await);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
// Terminal Task and its dispatch intent leave together: a half-done release is rolled back.
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(request.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(record)
}
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError> {
let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT));
let mut transaction = self.start().await?;
super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
.await
.map_err(|error| match error {
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
_ => TaskError::StateConflict,
})?;
let expired = sqlx::query(
"with expired as (
update tasks
set state = 'failed',
error_envelope = jsonb_build_object('reason', 'execution_deadline_elapsed'),
version = version + 1,
updated_at = now()
where task_id in (
select task_id from tasks
where state = 'queued' and execution_deadline <= now()
and lease_owner is null
order by task_id
limit $1
)
returning task_id
), released as (
delete from execution_admission_entries
where task_id in (select task_id from expired)
returning task_id
)
select (select count(*) from expired)::bigint as expired_queued",
)
.bind(batch)
.fetch_one(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let expired_queued: i64 = expired
.try_get("expired_queued")
.map_err(Self::unavailable)?;
let purged = sqlx::query(
"delete from tasks
where task_id in (
select task_id from tasks
where state in ('completed', 'failed', 'cancelled') and retain_until <= now()
order by task_id
limit $1
)",
)
.bind(batch)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(TaskPurgeReport {
expired_queued: u64::try_from(expired_queued).unwrap_or_default(),
expired_terminal: purged.rows_affected(),
})
}
async fn claim_batch(
&self,
owner: &str,
batch_limit: u32,
lease_ms: i64,
) -> Result<Vec<TaskLease>, TaskError> {
let batch = (batch_limit.clamp(1, 64)) as usize;
let mut transaction = self.start().await?;
// A checkpointed attempt is never redispatched blindly: once its lease expires another worker
// must decide per Task whether the pinned safe-retry evidence, the budget and the deadline
// still allow exactly one bounded new attempt (Story 4.4). Everything else resolves to the
// non-terminal uncertain state and releases its admission slot.
let candidates = sqlx::query(
"select t.task_id, t.current_attempt_id, t.attempts, t.max_attempts,
t.execution_deadline > clock_timestamp() as deadline_left,
coalesce(a.retry_allowed, false) as retry_allowed,
a.resolved_at is null as attempt_open
from tasks t
left join task_attempts a
on a.task_id = t.task_id and a.attempt_id = t.current_attempt_id
where t.state = 'working' and t.lease_owner is not null
and t.current_attempt_id is not null
and t.lease_until < clock_timestamp()
order by t.task_id
limit 64
for update of t skip locked",
)
.fetch_all(&mut *transaction)
.await
.map_err(Self::unavailable)?;
for candidate in candidates {
let task_id: Uuid = candidate.try_get("task_id").map_err(Self::unavailable)?;
let attempt_id: String = candidate
.try_get("current_attempt_id")
.map_err(Self::unavailable)?;
let attempts: i32 = candidate.try_get("attempts").map_err(Self::unavailable)?;
let max_attempts: i32 = candidate
.try_get("max_attempts")
.map_err(Self::unavailable)?;
let deadline_left: bool = candidate
.try_get("deadline_left")
.map_err(Self::unavailable)?;
let retry_allowed: bool = candidate
.try_get("retry_allowed")
.map_err(Self::unavailable)?;
let attempt_open: bool = candidate
.try_get("attempt_open")
.map_err(Self::unavailable)?;
let safe_recovery =
retry_allowed && attempt_open && attempts < max_attempts && deadline_left;
let close = sqlx::query(
"update task_attempts
set resolved_at = now(), outcome = 'outcome_unknown', certainty = 'unknown',
phase = 'dispatched'
where task_id = $1 and attempt_id = $2 and resolved_at is null",
)
.bind(task_id)
.bind(&attempt_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let _ = close;
if safe_recovery {
// Interrupted, not failed: re-arm the same admission intent and let the bounded claim
// hand the Task to a new owner with a new fence and exactly one new attempt.
sqlx::query(
"update tasks
set state = 'queued',
lease_owner = null,
lease_until = null,
current_attempt_id = null,
next_attempt_at = null,
version = version + 1,
updated_at = now()
where task_id = $1 and state = 'working'
and lease_until < clock_timestamp()",
)
.bind(task_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
sqlx::query(
"update execution_admission_entries
set state = 'reserved', granted_at = null
where task_id = $1 and state = 'executing'",
)
.bind(task_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
} else {
sqlx::query(
"update tasks
set state = 'outcome_unknown',
lease_owner = null,
lease_until = null,
error_envelope = jsonb_build_object('code', 'outcome_unknown', 'phase', 'dispatched'),
version = version + 1,
updated_at = now()
where task_id = $1 and state = 'working'
and lease_until < clock_timestamp()",
)
.bind(task_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(task_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
}
}
// A crash before the durable dispatch checkpoint left the Task queued for execution: its
// admission intent is re-armed as reserved and the Task becomes claimable again. Recovery is
// bounded by the <=30s lease expiry, inside the 60s story budget. The expiry condition is
// re-checked in the UPDATE so a concurrent renewal can never be clobbered by recovery.
sqlx::query(
"with candidates as (
select task_id from tasks
where state = 'working' and lease_owner is not null
and current_attempt_id is null
and lease_until < clock_timestamp()
order by task_id
limit 64
for update skip locked
), requeued as (
update tasks
set state = 'queued',
lease_owner = null,
lease_until = null,
next_attempt_at = null,
version = version + 1,
updated_at = now()
where task_id in (select task_id from candidates)
and state = 'working'
and current_attempt_id is null
and lease_until < clock_timestamp()
returning task_id
)
update execution_admission_entries
set state = 'reserved', granted_at = null
where task_id in (select task_id from requeued) and state = 'executing'",
)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
let mut leases = Vec::new();
// Continuation claim: after a committed answer the Task is `working` without a lease and still
// owns its executing admission permit. It is handed to exactly one new fence here, without a
// second admission grant, so waiting capacity is never double-counted.
let mut transaction = self.start().await?;
let resumed = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set lease_owner = $1,
lease_until = clock_timestamp() + ($2::bigint || ' milliseconds')::interval,
fence = fence + 1,
updated_at = now()
where task_id in (
select t.task_id
from tasks t
join task_input_requests r on r.task_id = t.task_id and r.accepted_at is not null
join execution_admission_entries e
on e.task_id = t.task_id and e.state = 'executing'
where t.state = 'working' and t.lease_owner is null
and t.current_attempt_id is null
order by t.task_id
limit $3
for update of t skip locked
)
returning {TASK_PROJECTION}"
)))
.bind(owner)
.bind(lease_ms)
.bind(i64::try_from(batch.clamp(1, 64)).unwrap_or(64))
.fetch_all(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
for row in resumed {
let record = record_from_row(&row).map_err(Self::unavailable)?;
leases.push(TaskLease {
owner: owner.to_owned(),
fence: record.fence,
lease_until: record.lease_until_unix_ms.unwrap_or_default(),
record,
});
}
let remaining = batch.saturating_sub(leases.len());
for _ in 0..remaining {
let mut transaction = self.start().await?;
let workspace_cursor =
match super::admission::lock_admission_in_tx(&mut transaction, "*").await {
Ok(cursor) => cursor,
Err(error) => {
transaction.rollback().await.ok();
return Err(TaskError::unavailable(format!("{error:?}")));
}
};
let (task_id, _updated_cursor) = match super::admission::grant_next_task(
&mut transaction,
"*",
&workspace_cursor,
&self.limits,
self.scheduler,
)
.await
{
Ok(value) => value,
Err(error) => {
transaction.rollback().await.ok();
return Err(TaskError::unavailable(format!("{error:?}")));
}
};
// Persist the updated controller cursor even when no task is granted.
let Some(task_id) = task_id else {
transaction.commit().await.map_err(Self::unavailable)?;
break;
};
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = 'working',
lease_owner = $1,
lease_until = clock_timestamp() + ($2::bigint || ' milliseconds')::interval,
fence = fence + 1,
updated_at = now()
where task_id = $3
and (
(state = 'queued' and lease_owner is null)
or (state = 'working' and lease_owner is not null
and lease_until < clock_timestamp() and current_attempt_id is null)
)
and (next_attempt_at is null or next_attempt_at <= clock_timestamp())
returning {TASK_PROJECTION}"
)))
.bind(owner)
.bind(lease_ms)
.bind(task_id)
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
// Persist cursor update (grant_next_task already did it) regardless.
let Some(row) = row else {
transaction.rollback().await.ok();
break;
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
leases.push(TaskLease {
owner: owner.to_owned(),
fence: record.fence,
lease_until: record.lease_until_unix_ms.unwrap_or_default(),
record,
});
transaction.commit().await.map_err(Self::unavailable)?;
}
Ok(leases)
}
async fn renew_lease(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
lease_ms: i64,
) -> Result<TaskLease, TaskError> {
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set lease_until = clock_timestamp() + ($1::bigint || ' milliseconds')::interval,
updated_at = now()
where task_id = $2 and lease_owner = $3 and fence = $4
and lease_until > clock_timestamp()
returning {TASK_PROJECTION}"
)))
.bind(lease_ms)
.bind(task_id.as_uuid())
.bind(owner)
.bind(i64::try_from(fence).unwrap_or_default())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
let lease = TaskLease {
owner: owner.to_owned(),
fence: record.fence,
lease_until: record.lease_until_unix_ms.unwrap_or_default(),
record,
};
transaction.commit().await.map_err(Self::unavailable)?;
Ok(lease)
}
async fn write_progress(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
progress: TaskEnvelope,
) -> Result<TaskRecord, TaskError> {
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set progress_envelope = $1, updated_at = now()
where task_id = $2 and lease_owner = $3 and fence = $4
and lease_until > clock_timestamp()
returning {TASK_PROJECTION}"
)))
.bind(progress.value().clone())
.bind(task_id.as_uuid())
.bind(owner)
.bind(i64::try_from(fence).unwrap_or_default())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(record)
}
async fn read_executable_input(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
) -> Result<(String, String, i64), TaskError> {
let row = sqlx::query(
"select executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch
from tasks
where task_id = $1 and state = 'working' and lease_owner = $2 and fence = $3
and lease_until > clock_timestamp()",
)
.bind(task_id.as_uuid())
.bind(owner)
.bind(i64::try_from(fence).unwrap_or_default())
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::StateConflict);
};
let ciphertext: Option<String> = row
.try_get("executable_input_ciphertext")
.map_err(Self::unavailable)?;
let key_version: Option<String> = row
.try_get("executable_input_key_version")
.map_err(Self::unavailable)?;
let epoch: Option<i64> = row
.try_get("executable_input_master_key_epoch")
.map_err(Self::unavailable)?;
if let Some(epoch) = epoch {
let active = sqlx::query_scalar::<_, i64>(
"select epoch from master_key_identities where status = 'active' order by epoch desc limit 1",
)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
if let Some(active) = active
&& active != epoch
{
return Err(TaskError::StateConflict);
}
}
match (ciphertext, key_version, epoch) {
(Some(ciphertext), Some(key_version), Some(epoch)) if !ciphertext.is_empty() => {
Ok((ciphertext, key_version, epoch))
}
_ => Err(TaskError::NotFound),
}
}
async fn checkpoint_dispatch(&self, request: BeginAttempt) -> Result<AttemptLease, TaskError> {
self.checkpoint_dispatch_attempt(request).await
}
async fn resolve_attempt(
&self,
lease: &AttemptLease,
outcome: AttemptOutcome,
) -> Result<AttemptResolution, TaskError> {
self.resolve_attempt_outcome(lease, outcome).await
}
async fn cancel(&self, request: TaskCancelRequest) -> Result<TaskCancelOutcome, TaskError> {
self.cancel_task(request).await
}
async fn attempts(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<Vec<TaskAttempt>, TaskError> {
self.attempt_history(access, task_id).await
}
async fn finish_leased(
&self,
request: TaskFinishRequest,
owner: &str,
fence: u64,
) -> Result<TaskRecord, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
let access = &request.access;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = $1,
result_envelope = $2,
error_envelope = $3,
version = version + 1,
updated_at = now(),
lease_owner = null,
lease_until = null
where task_id = $4 and workspace_id = $5 and owner_kind = $6
and owner_id = $7 and authorization_scope = $8
and state = $9 and version = $10
and lease_owner = $11 and fence = $12
and lease_until > clock_timestamp()
and current_attempt_id is null
returning {TASK_PROJECTION}"
)))
.bind(request.target_state.as_str())
.bind(request.result.as_ref().map(|value| value.value().clone()))
.bind(request.error.as_ref().map(|value| value.value().clone()))
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.bind(request.expected_state.as_str())
.bind(i64::from(request.expected_version))
.bind(owner)
.bind(i64::try_from(fence).unwrap_or_default())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(request.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(record)
}
async fn refuse_attempt(&self, lease: &TaskLease, code: &str) -> Result<TaskRecord, TaskError> {
self.refuse_attempt_resolution(lease, code).await
}
}
fn policy_mode_str(mode: ToolExecutionMode) -> &'static str {
match mode {
ToolExecutionMode::Synchronous => "synchronous",
ToolExecutionMode::Background => "background",
ToolExecutionMode::Adaptive => "adaptive",
}
}
fn task_sql_error(error: sqlx::Error) -> AttemptError {
if error
.as_database_error()
.and_then(|database| database.code())
.as_deref()
== Some("23505")
{
return AttemptError::UniqueViolation;
}
AttemptError::from_sql(error)
}
fn record_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskRecord, TaskError> {
let task_id: Uuid = row.try_get("task_id").map_err(unavailable_row)?;
let state: String = row.try_get("state").map_err(unavailable_row)?;
let owner_kind: String = row.try_get("owner_kind").map_err(unavailable_row)?;
let policy_mode: String = row.try_get("policy_mode").map_err(unavailable_row)?;
let trace_id: String = row.try_get("trace_id").map_err(unavailable_row)?;
let request_id: String = row.try_get("request_id").map_err(unavailable_row)?;
let envelope = |column: &str| -> Result<Option<TaskEnvelope>, TaskError> {
let value: Option<Value> = row.try_get(column).map_err(unavailable_row)?;
value.map(TaskEnvelope::from_persisted).transpose()
};
Ok(TaskRecord {
task_id: TaskId::parse(&task_id.to_string())?,
state: TaskState::parse(&state)?,
workspace_id: row.try_get("workspace_id").map_err(unavailable_row)?,
owner_kind: TaskOwnerKind::parse(&owner_kind)?,
owner_id: row.try_get("owner_id").map_err(unavailable_row)?,
authorization_scope: row
.try_get("authorization_scope")
.map_err(unavailable_row)?,
platform_api_key_id: row
.try_get::<Option<String>, _>("platform_api_key_id")
.map_err(unavailable_row)?
.map(PlatformApiKeyId::new),
operation_id: row.try_get("operation_id").map_err(unavailable_row)?,
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
effective_input_fingerprint: row
.try_get("effective_input_fingerprint")
.map_err(unavailable_row)?,
pins: TaskPins {
operation_version: u32::try_from(
row.try_get::<i32, _>("operation_version")
.map_err(unavailable_row)?,
)
.unwrap_or_default(),
policy_mode: match policy_mode.as_str() {
"background" => ToolExecutionMode::Background,
"adaptive" => ToolExecutionMode::Adaptive,
_ => ToolExecutionMode::Synchronous,
},
execution_deadline_unix_ms: row
.try_get("execution_deadline_ms")
.map_err(unavailable_row)?,
retain_until_unix_ms: row.try_get("retain_until_ms").map_err(unavailable_row)?,
approval_reference: row.try_get("approval_reference").map_err(unavailable_row)?,
retry: TaskRetryPolicy::new(
u32::try_from(
row.try_get::<i32, _>("max_attempts")
.map_err(unavailable_row)?,
)
.unwrap_or(1),
row.try_get::<i64, _>("retry_backoff_step_ms")
.map_err(unavailable_row)?,
),
},
support: TaskSupportIds {
request_id: crank_core::RequestId::parse(&request_id)
.map_err(|_| TaskError::unavailable("stored request id is invalid"))?,
trace_id: crank_core::TraceId::parse(&trace_id)
.map_err(|_| TaskError::unavailable("stored trace id is invalid"))?,
traceparent: row.try_get("traceparent").map_err(unavailable_row)?,
trace_sampled: row.try_get("trace_sampled").map_err(unavailable_row)?,
},
input: envelope("input_envelope")?
.ok_or_else(|| TaskError::unavailable("stored task input envelope is missing"))?,
progress: envelope("progress_envelope")?,
result: envelope("result_envelope")?,
error: envelope("error_envelope")?,
attempts: u32::try_from(row.try_get::<i32, _>("attempts").map_err(unavailable_row)?)
.unwrap_or_default(),
fence: u64::try_from(row.try_get::<i64, _>("fence").map_err(unavailable_row)?)
.unwrap_or_default(),
lease_owner: row.try_get("lease_owner").map_err(unavailable_row)?,
lease_until_unix_ms: row.try_get("lease_until_ms").map_err(unavailable_row)?,
current_attempt_id: row.try_get("current_attempt_id").map_err(unavailable_row)?,
dispatch_started_at_unix_ms: row
.try_get("dispatch_started_at_ms")
.map_err(unavailable_row)?,
next_attempt_at_unix_ms: row.try_get("next_attempt_at_ms").map_err(unavailable_row)?,
cancel_requested_at_unix_ms: row
.try_get("cancel_requested_at_ms")
.map_err(unavailable_row)?,
cancel_reason: row.try_get("cancel_reason").map_err(unavailable_row)?,
version: u32::try_from(row.try_get::<i32, _>("version").map_err(unavailable_row)?)
.unwrap_or_default(),
created_at_unix_ms: row.try_get("created_ms").map_err(unavailable_row)?,
updated_at_unix_ms: row.try_get("updated_ms").map_err(unavailable_row)?,
})
}
fn unavailable_row(error: sqlx::Error) -> TaskError {
PostgresTaskStore::unavailable(error)
}
@@ -2,6 +2,7 @@ use crank_registry::{MigrationAuthority, MigrationPreflight, PostgresRegistry};
use sqlx::Row;
mod artifact_metadata;
mod durable_tasks_guard;
mod import_plan;
mod legacy_adoption;
mod preflight;
@@ -72,6 +73,11 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
(23, "prompt-dependencies-v23"),
(24, "execution-admission-v24"),
(25, "durable-tasks-v25"),
(26, "durable-tasks-v26"),
(27, "durable-tasks-v27"),
(28, "durable-tasks-v28"),
(29, "durable-tasks-v29"),
(30, "durable-task-input-v30"),
];
assert_eq!(rows.len(), expected.len());
for (row, (version, name)) in rows.iter().zip(expected) {
@@ -182,7 +188,7 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
);
assert_eq!(
MigrationAuthority::preflight(first.pool()).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 30 }
);
}
#[tokio::test]
@@ -230,7 +236,7 @@ async fn v18_guard_rejects_missing_agent_resource_binding_trigger() {
.expect("the V18-local guard must be restorable");
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 30 },
"restoring the exact V18 contract must return the current verdict"
);
}
@@ -356,7 +362,7 @@ async fn ledgerless_legacy_baseline_is_consolidated_without_data_loss() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 25,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -469,13 +475,13 @@ async fn healthy_v2_is_reported_as_migration_required_and_upgrades_to_v3() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 2,
target: 25,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 30 }
);
let trace_column: bool = sqlx::query_scalar(
"select exists (
@@ -518,7 +524,7 @@ async fn healthy_v3_upgrades_to_v4_with_honest_legacy_snapshot_provenance() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 3,
target: 25,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -578,7 +584,7 @@ async fn healthy_v4_upgrades_to_v5_without_fabricating_legacy_outcomes() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 4,
target: 25,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -832,6 +838,6 @@ async fn v3_upgrade_ignores_oversized_legacy_request_ids_in_partial_index() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 30 }
);
}
@@ -32,7 +32,7 @@ async fn healthy_v11_upgrades_to_v13_without_rewriting_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 11,
target: 25,
target: 30,
}
);
@@ -177,7 +177,7 @@ async fn v12_guard_still_rejects_column_constraint_index_and_relation_drift() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 30 },
"restore: {restore}"
);
}
@@ -230,7 +230,7 @@ async fn v13_guard_rejects_wrong_cleanup_index_order_and_predicate() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 30 },
"restore: {restore}"
);
}
@@ -507,7 +507,7 @@ async fn assert_v12_drift(pool: &sqlx::PgPool, drift: &str, restore: &str) {
.unwrap();
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 30 },
"restore: {restore}"
);
}
@@ -845,7 +845,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 30 }
);
for (drift, restore) in [
@@ -879,7 +879,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 30 },
"restore: {restore}"
);
}
@@ -289,7 +289,7 @@ async fn assert_primary_key_nullability(pool: &sqlx::PgPool, table: &str, column
);
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 30 },
"a rejected primary-key nullability change must leave the schema current"
);
@@ -473,7 +473,7 @@ async fn assert_definition_only_drift(
assert_eq!(super::catalog_constraint_definitions(pool).await, before);
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 30 }
);
}
@@ -554,7 +554,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 30 },
"the baseline before the drift matrix must be the current V16 schema"
);
let mut observations: Vec<String> = Vec::new();
@@ -773,7 +773,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
.await
.expect("restoring the exact definition must apply");
let restored = MigrationAuthority::preflight(&pool).await;
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 25 }));
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 30 }));
// Точное восстановление доказывается сравнением снятого снимка с baseline, а не
// мнением того же guard-а, который принимает неправильные определения (F4).
let restored_exact = catalog_definition_snapshot(&pool).await == before;
@@ -804,7 +804,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 30 },
"baseline before inserting the shadow schema"
);
@@ -875,7 +875,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
// Тень не меняет вердикт guard-а по текущей схеме.
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 30 },
"a same-named object in another schema must not change the verdict for the current schema"
);
@@ -947,7 +947,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 30 },
"the exact restoration must return the current V16 schema"
);
@@ -958,7 +958,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 30 }
);
assert!(
@@ -0,0 +1,83 @@
use super::*;
use crank_registry::MigrationApplyResult;
#[tokio::test]
async fn clean_legacy_ledger_upgrades_to_v28_after_v25_removal() {
let database_url = crank_test_support::postgres_schema_url("test_v43_clean_legacy").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
super::remove_v25_schema(&pool).await;
// remove_v25_schema drops `tasks` and ledger >= 25, but V25 also added the
// nullable dispatch-intent column/index/FK to execution_admission_entries;
// restore those to the V24 shape so the legacy ledger is self-consistent.
sqlx::raw_sql(
"alter table execution_admission_entries drop constraint if exists execution_admission_entries_task_fkey;
drop index if exists execution_admission_entries_task_uidx;
alter table execution_admission_entries drop column if exists task_id;",
)
.execute(&pool)
.await
.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 24,
target: 30,
}
);
assert_eq!(
MigrationAuthority::apply(&pool).await.unwrap(),
MigrationApplyResult::Applied { from: 24, to: 30 }
);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 30 }
);
}
#[tokio::test]
async fn partial_v27_on_legacy_ledger_is_rejected_by_preflight() {
let database_url = crank_test_support::postgres_schema_url("test_v43_partial_v27").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
super::remove_v25_schema(&pool).await;
sqlx::raw_sql(
"alter table execution_admission_entries drop constraint if exists execution_admission_entries_task_fkey;
drop index if exists execution_admission_entries_task_uidx;
alter table execution_admission_entries drop column if exists task_id;",
)
.execute(&pool)
.await
.unwrap();
// Reintroduce a partial V27-shaped `tasks` table on top of the legacy (v24) ledger.
sqlx::raw_sql(sqlx::AssertSqlSafe(
"create table tasks (
task_id uuid primary key,
executable_input_ciphertext text,
executable_input_key_version text,
executable_input_master_key_epoch bigint
);",
))
.execute(&pool)
.await
.unwrap();
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
assert_eq!(error.code(), "partial_sequence");
// The rejection today is attributed to V25's absent-schema guard because the
// `tasks` relation already exists, so a behavioral RED pin for the missing
// V27 guard line cannot be reached; this test fixes the rejection contract
// instead of claiming new behavior.
assert_eq!(error.version(), Some(24));
assert_eq!(error.stage(), "preflight.schema");
let count: i64 =
sqlx::query_scalar("select count(*)::bigint from __crank_migrations where version >= 25")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(
count, 0,
"nothing at or above V25 should have been applied over the partial fixture"
);
}
@@ -71,7 +71,7 @@ async fn fresh_install_exposes_import_plan_objects_with_cascading_job_fk() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 30 }
);
let tables = sqlx::query(
"select table_name
@@ -224,7 +224,7 @@ async fn healthy_v13_upgrades_to_v14_without_changing_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 13,
target: 25,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -88,7 +88,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 25,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -101,7 +101,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
assert_eq!(operation_count, 1);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 30 }
);
}
@@ -194,7 +194,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 25,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -230,7 +230,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 30 }
);
}
@@ -262,7 +262,7 @@ async fn published_initial_mcp_layout_is_accepted() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 25,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -280,7 +280,7 @@ async fn published_initial_mcp_layout_is_accepted() {
assert!(supports_elicitation);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 30 }
);
}
@@ -314,13 +314,13 @@ async fn published_in_place_mcp_upgrade_layout_is_accepted() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 25,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 30 }
);
}
@@ -14,7 +14,7 @@ async fn fresh_install_creates_the_empty_revocation_authority() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 30 }
);
let rows: i64 = sqlx::query_scalar("select count(*) from resource_security_revocations")
.fetch_one(&pool)
@@ -42,12 +42,12 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 18,
target: 25,
target: 30,
}
);
assert_eq!(
MigrationAuthority::apply(&pool).await.unwrap(),
MigrationApplyResult::Applied { from: 18, to: 25 }
MigrationApplyResult::Applied { from: 18, to: 30 }
);
let after: Vec<(i64, String, String)> =
sqlx::query_as("select version, name, checksum from __crank_migrations where version < 19 order by version")
@@ -57,7 +57,7 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
assert_eq!(before, after, "V16–V18 ledger rows must not be rewritten");
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 30 }
);
}
@@ -86,16 +86,22 @@ async fn a_database_newer_than_the_binary_fails_closed_at_startup() {
MigrationAuthority::apply(&pool).await.unwrap();
// The same `version > CURRENT_VERSION` branch an older binary takes when it meets a newer
// database; a synthetic version above the current one exercises it without a second binary.
let current: i64 = sqlx::query_scalar("select max(version) from __crank_migrations")
.fetch_one(&pool)
.await
.unwrap();
let future = current + 1;
sqlx::query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values (26, 'future-v26', repeat('a', 64), 'expand', 'n-minus-one-readable')",
values ($1, 'future-synthetic', repeat('a', 64), 'expand', 'n-minus-one-readable')",
)
.bind(future)
.execute(&pool)
.await
.unwrap();
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
assert_eq!(error.code(), "future_version");
assert_eq!(error.version(), Some(26));
assert_eq!(error.version(), Some(future));
assert_eq!(error.recovery(), "install_matching_application");
assert!(
MigrationAuthority::require_current(&pool).await.is_err(),
@@ -155,7 +161,7 @@ async fn failed_v19_migration_rolls_back_schema_and_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 18,
target: 25,
target: 30,
}
);
}
@@ -63,7 +63,7 @@ async fn failed_artifact_metadata_migration_rolls_back_schema_and_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 11,
target: 25,
target: 30,
}
);
}
@@ -330,9 +330,14 @@ pub(crate) async fn remove_v12_schema(pool: &sqlx::PgPool) {
/// Story 4.2: снять V25 (durable Tasks и Task-linked dispatch intent) перед возвратом к < 25.
pub(crate) async fn remove_v25_schema(pool: &sqlx::PgPool) {
// V25 created `tasks`; V26–V28 only altered it, so dropping the table removes their schema too.
// V30 adds tables that reference `tasks`; they are dropped explicitly so the legacy preflight
// sees an actually absent V30 shape instead of orphan continuation tables.
sqlx::raw_sql(
"drop table if exists tasks cascade;
delete from __crank_migrations where version = 25;",
"drop table if exists task_input_audit cascade;
drop table if exists task_input_requests cascade;
drop table if exists tasks cascade;
delete from __crank_migrations where version >= 25;",
)
.execute(pool)
.await
@@ -8,14 +8,19 @@
use super::common;
use crank_core::{
AdmissionMode, AdmissionOutcome, AdmissionRequest, AdmissionScope, ApprovalEvidence,
ExecutionAdmission, NoTasksFallback, RequestId, TaskAcceptRequest, TaskAccess, TaskEnvelope,
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore,
TaskSupportIds, ToolExecutionMode, TraceContext, current_unix_ms,
AdmissionMode, AdmissionOutcome, AdmissionRequest, AdmissionScope, AgentId, ApprovalEvidence,
BeginAttempt, ExecutionAdmission, NoTasksFallback, PlatformApiKey, PlatformApiKeyId,
PlatformApiKeyKind, PlatformApiKeyScope, PlatformApiKeyStatus, RequestId, TaskAcceptRequest,
TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins,
TaskState, TaskStore, TaskSupportIds, ToolExecutionMode, TraceContext, WorkspaceId,
current_unix_ms,
};
use crank_registry::{
AdmissionLimits, CreatePlatformApiKeyRequest, PostgresExecutionAdmission, PostgresTaskStore,
};
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission, PostgresTaskStore};
use serde_json::json;
use sqlx::{PgPool, Row};
use time::{Duration as TimeDuration, OffsetDateTime};
const WORKSPACE: &str = "ws_default";
const AGENT: &str = "agent_a";
@@ -48,6 +53,7 @@ fn request(key: &str, fingerprint: &str, agent: &str, deadline_in_ms: i64) -> Ta
owner_kind: TaskOwnerKind::Agent,
owner_id: agent.to_owned(),
authorization_scope: format!("credential:pk_{agent}"),
platform_api_key_id: Some(PlatformApiKeyId::new(format!("pk_{agent}"))),
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 3,
@@ -55,11 +61,15 @@ fn request(key: &str, fingerprint: &str, agent: &str, deadline_in_ms: i64) -> Ta
execution_deadline_unix_ms: deadline,
retain_until_unix_ms: deadline + 3_600_000,
approval_reference: Some("appr_1".to_owned()),
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: key.to_owned(),
input_fingerprint: fingerprint.to_owned(),
input: TaskEnvelope::try_redacted(&json!({"report": "q3", "api_key": "sk_live"})).unwrap(),
support: unsampled_support(),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
}
}
@@ -664,6 +674,80 @@ async fn invalid_bounded_envelope_is_rejected_before_persistence() {
database.cleanup().await;
}
#[tokio::test]
async fn task_worker_lease_and_fencing_contract() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let created = store
.accept(request(&key("lease1"), &digest("f1"), AGENT, 30_000))
.await
.unwrap();
let leases = store.claim_batch("worker_1", 4, 10_000).await.unwrap();
assert_eq!(leases.len(), 1);
assert_eq!(leases[0].record.task_id, created.task_id);
assert_eq!(leases[0].record.state, TaskState::Working);
assert_eq!(leases[0].owner, "worker_1");
let lease = &leases[0];
let fenced_progress = store
.write_progress(
"worker_1",
lease.record.task_id,
lease.fence,
TaskEnvelope::try_redacted(&json!({"progress": 1})).unwrap(),
)
.await
.unwrap();
assert!(fenced_progress.progress.is_some());
// stale fence rejected
let bad = store
.renew_lease("worker_1", lease.record.task_id, lease.fence + 1, 10_000)
.await;
assert!(bad.is_err());
// checkpoint dispatch once
let checkpointed = store
.checkpoint_dispatch(BeginAttempt {
task_id: lease.record.task_id,
owner: "worker_1".to_owned(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: "attempt-1".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: true,
})
.await
.unwrap();
assert_eq!(checkpointed.attempt.attempt_id, "attempt-1");
assert_eq!(checkpointed.attempt.ordinal, 1);
assert!(checkpointed.task.dispatch_started_at_unix_ms.is_some());
assert_eq!(
checkpointed.task.current_attempt_id.as_deref(),
Some("attempt-1")
);
// second checkpoint rejected
let second = store
.checkpoint_dispatch(BeginAttempt {
task_id: lease.record.task_id,
owner: "worker_1".to_owned(),
fence: lease.fence,
expected_version: checkpointed.task.version,
attempt_id: "attempt-2".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: true,
})
.await;
assert!(second.is_err());
database.cleanup().await;
}
#[tokio::test]
async fn store_failure_fails_closed_without_records() {
let database = common::TestDatabase::new().await;
@@ -680,3 +764,155 @@ async fn store_failure_fails_closed_without_records() {
database.cleanup().await;
}
/// Story 4.3 ATDD: the typed immutable Platform API key reference of an accepted Task is persisted,
/// read back through the lease, and only an active, unexpired, exactly bound key is authoritatively
/// revalidated. Revocation, expiry and a foreign workspace/agent binding all close; an administrator
/// Task stores no machine key identity at all, so a worker fails closed on it.
#[tokio::test]
async fn accepted_task_authority_reference_is_persisted_and_revalidated() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let workspace = WorkspaceId::new(WORKSPACE);
let now = OffsetDateTime::now_utc();
// `ws_default` is seeded by the baseline migration; the keys only need that workspace row.
let agent = AgentId::new(AGENT);
let active_id = PlatformApiKeyId::new("pk_task_active");
for key in [
authority_key(
"pk_task_active",
&workspace,
AGENT,
PlatformApiKeyStatus::Active,
None,
),
authority_key(
"pk_task_revoked",
&workspace,
AGENT,
PlatformApiKeyStatus::Revoked,
None,
),
authority_key(
"pk_task_expired",
&workspace,
AGENT,
PlatformApiKeyStatus::Active,
Some(now - TimeDuration::hours(1)),
),
] {
let secret_hash = format!("secret-hash-{}", key.id.as_str());
registry
.create_platform_api_key(CreatePlatformApiKeyRequest {
secret_hash: &secret_hash,
api_key: &key,
})
.await
.unwrap();
}
// Acceptance persists only the immutable typed reference, never the bearer secret.
let mut accepted = request(&key("k_authority"), &digest("fa"), AGENT, 300_000);
accepted.platform_api_key_id = Some(active_id.clone());
let task_id = accepted.task_id;
store.accept(accepted).await.expect("acceptance commits");
let leases = store
.claim_batch("worker_authority", 8, 30_000)
.await
.expect("claim");
let lease = leases
.into_iter()
.find(|lease| lease.record.task_id == task_id)
.expect("accepted task is leased");
assert_eq!(lease.record.platform_api_key_id.as_ref(), Some(&active_id));
// Positive: the exact active key of this workspace/agent is revalidated authoritatively.
let revalidated = registry
.revalidate_platform_api_key(&workspace, &agent, &active_id)
.await
.expect("authoritative read")
.expect("active key is authoritative");
assert!(
revalidated
.api_key
.scopes
.contains(&PlatformApiKeyScope::Write)
);
// Negatives: revocation, expiry and a foreign workspace/agent binding all fail closed.
let foreign_agent = AgentId::new("agent_b");
let foreign_workspace = WorkspaceId::new("ws_elsewhere");
for (other_workspace, other_agent, other_key) in [
(&workspace, &agent, PlatformApiKeyId::new("pk_task_revoked")),
(&workspace, &agent, PlatformApiKeyId::new("pk_task_expired")),
(&workspace, &foreign_agent, active_id.clone()),
(&foreign_workspace, &agent, active_id.clone()),
] {
let found = registry
.revalidate_platform_api_key(other_workspace, other_agent, &other_key)
.await
.unwrap();
assert!(
found.is_none(),
"authority must fail closed for {}",
other_key.as_str()
);
}
// An administrator Task carries no machine key identity: the worker has nothing to revalidate.
let mut admin_request = request(&key("k_admin"), &digest("fb"), AGENT, 300_000);
admin_request.owner_kind = TaskOwnerKind::Admin;
admin_request.owner_id = "user_admin".to_owned();
admin_request.authorization_scope = "admin:user_admin".to_owned();
admin_request.platform_api_key_id = None;
let admin_task = admin_request.task_id;
store
.accept(admin_request)
.await
.expect("administrator acceptance commits");
let admin_record = store
.read(
&TaskAccess {
workspace_id: WORKSPACE.to_owned(),
owner_kind: TaskOwnerKind::Admin,
owner_id: "user_admin".to_owned(),
authorization_scope: "admin:user_admin".to_owned(),
},
admin_task,
)
.await
.expect("owner read");
assert_eq!(admin_record.platform_api_key_id, None);
database.cleanup().await;
}
fn authority_key(
id: &str,
workspace_id: &WorkspaceId,
agent: &str,
status: PlatformApiKeyStatus,
expires_at: Option<OffsetDateTime>,
) -> PlatformApiKey {
PlatformApiKey {
id: PlatformApiKeyId::new(id),
workspace_id: workspace_id.clone(),
agent_id: Some(AgentId::new(agent)),
key_kind: PlatformApiKeyKind::McpClient,
name: id.to_owned(),
prefix: "crk_live".to_owned(),
scopes: vec![PlatformApiKeyScope::Write],
status,
created_at: OffsetDateTime::now_utc(),
last_used_at: None,
expires_at,
allowed_origins: Vec::new(),
}
}
mod task_attempts;
mod task_worker;
@@ -0,0 +1,534 @@
use super::*;
use crank_core::{
AttemptOutcome, AttemptOutcomeClass, AttemptPhase, AttemptResolution, BeginAttempt,
TaskCancelOutcome, TaskCancelRequest, TaskRetryPolicy,
};
use serde_json::json;
#[tokio::test]
async fn cancel_before_checkpoint_wins_race_with_claim_and_never_dispatches() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let created = store
.accept(request(
&key("cancel_before_1"),
&digest("fb1"),
AGENT,
30_000,
))
.await
.unwrap();
let outcome = store
.cancel(TaskCancelRequest {
access: access(AGENT),
task_id: created.task_id,
expected_version: None,
reason: "cancel before dispatch".to_owned(),
})
.await
.unwrap();
match outcome {
TaskCancelOutcome::Cancelled(record) => assert_eq!(record.state, TaskState::Cancelled),
other => panic!("expected Cancelled, got {other:?}"),
}
let claims = store.claim_batch("worker", 4, 10_000).await.unwrap();
assert!(claims.is_empty(), "cancelled task must not be claimed");
let attempts = store
.attempts(&access(AGENT), created.task_id)
.await
.unwrap();
assert!(attempts.is_empty(), "no attempts for cancelled task");
let intents: i64 = sqlx::query_scalar(
"select count(*)::bigint from execution_admission_entries where task_id = $1",
)
.bind(created.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(intents, 0, "dispatch intent must be removed");
database.cleanup().await;
}
#[tokio::test]
async fn cancel_after_checkpoint_is_uncertain_and_never_cancelled_or_retried() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let created = store
.accept(request(
&key("cancel_after_1"),
&digest("fa1"),
AGENT,
30_000,
))
.await
.unwrap();
let leases = store.claim_batch("worker", 1, 10_000).await.unwrap();
let lease = leases.into_iter().next().expect("claim");
let leased = store
.checkpoint_dispatch(BeginAttempt {
task_id: lease.record.task_id,
owner: "worker".to_owned(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: "attempt-cancel-after".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: true,
})
.await
.expect("checkpoint");
let outcome = store
.cancel(TaskCancelRequest {
access: access(AGENT),
task_id: created.task_id,
expected_version: None,
reason: "cancel after checkpoint".to_owned(),
})
.await
.unwrap();
match outcome {
TaskCancelOutcome::OutcomeUnknown(record) => {
assert_eq!(record.state, TaskState::OutcomeUnknown)
}
other => panic!("expected OutcomeUnknown, got {other:?}"),
}
let record = store.read(&access(AGENT), created.task_id).await.unwrap();
assert_eq!(record.state, TaskState::OutcomeUnknown);
let attempts = store
.attempts(&access(AGENT), created.task_id)
.await
.unwrap();
assert_eq!(attempts.len(), 1);
assert_eq!(
attempts[0].outcome,
Some(AttemptOutcomeClass::OutcomeUnknown)
);
// Repeated cancel stays uncertain.
let outcome2 = store
.cancel(TaskCancelRequest {
access: access(AGENT),
task_id: created.task_id,
expected_version: None,
reason: "cancel after checkpoint".to_owned(),
})
.await
.unwrap();
assert!(matches!(outcome2, TaskCancelOutcome::OutcomeUnknown(_)));
// finish_leased by the original lease must be rejected.
let finish = store
.finish_leased(
TaskFinishRequest {
access: access(AGENT),
task_id: created.task_id,
expected_state: leased.task.state,
expected_version: leased.task.version,
target_state: TaskState::Completed,
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
error: None,
},
"worker",
leased.attempt.fence,
)
.await;
assert!(matches!(finish, Err(TaskError::StateConflict)));
let claims2 = store.claim_batch("worker2", 4, 10_000).await.unwrap();
assert!(
!claims2.iter().any(|l| l.record.task_id == created.task_id),
"outcome unknown task must never be claimed"
);
database.cleanup().await;
}
#[tokio::test]
async fn checkpoint_and_resolve_are_cas_guarded_and_history_is_immutable() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let created = store
.accept(request(&key("cas_1"), &digest("fc1"), AGENT, 30_000))
.await
.unwrap();
let leases = store.claim_batch("worker", 1, 10_000).await.unwrap();
let lease = leases.into_iter().next().expect("claim");
let first_version = lease.record.version;
let committed = store
.checkpoint_dispatch(BeginAttempt {
task_id: lease.record.task_id,
owner: "worker".to_owned(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: "attempt-cas".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await
.expect("first checkpoint");
let committed_lease = committed;
// Repeated checkpoint with the same old version -> StateConflict.
let second = store
.checkpoint_dispatch(BeginAttempt {
task_id: lease.record.task_id,
owner: "worker".to_owned(),
fence: lease.fence,
expected_version: first_version,
attempt_id: "attempt-cas-2".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await;
assert!(matches!(second, Err(TaskError::StateConflict)));
// resolve with a modified lease task version -> StateConflict.
let mut bad_lease = committed_lease.clone();
bad_lease.task.version = first_version;
let bad_resolve = store
.resolve_attempt(
&bad_lease,
AttemptOutcome::Completed(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
)
.await;
assert!(matches!(bad_resolve, Err(TaskError::StateConflict)));
// Correct resolve -> Finished(Completed).
let ok = store
.resolve_attempt(
&committed_lease,
AttemptOutcome::Completed(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
)
.await
.expect("resolve");
match ok {
AttemptResolution::Finished(rec) => assert_eq!(rec.state, TaskState::Completed),
other => panic!("expected Finished, got {other:?}"),
}
// Repeated resolve -> StateConflict.
let repeat = store
.resolve_attempt(
&committed_lease,
AttemptOutcome::Completed(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
)
.await;
assert!(matches!(repeat, Err(TaskError::StateConflict)));
let attempts = store
.attempts(&access(AGENT), created.task_id)
.await
.unwrap();
assert_eq!(attempts.len(), 1, "exactly one immutable attempt record");
assert_eq!(attempts[0].outcome, Some(AttemptOutcomeClass::Completed));
let _ = created;
database.cleanup().await;
}
#[tokio::test]
async fn expired_checkpointed_attempt_recovers_only_with_safe_retry_evidence() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
// Task A: retry policy allows 2 attempts, retry_allowed=true.
let mut req_a = request(&key("rec_a"), &digest("fa"), AGENT, 30_000);
req_a.pins.retry = TaskRetryPolicy::new(2, 0);
let created_a = store.accept(req_a).await.unwrap();
// Task B: single attempt, retry_allowed=false.
let created_b = store
.accept(request(&key("rec_b"), &digest("fb"), AGENT, 30_000))
.await
.unwrap();
let claims = store.claim_batch("worker_dead", 2, 150).await.unwrap();
assert_eq!(claims.len(), 2);
let lease_a = claims
.iter()
.find(|l| l.record.task_id == created_a.task_id)
.cloned()
.expect("claim A");
let lease_b = claims
.iter()
.find(|l| l.record.task_id == created_b.task_id)
.cloned()
.expect("claim B");
store
.checkpoint_dispatch(BeginAttempt {
task_id: lease_b.record.task_id,
owner: "worker_dead".to_owned(),
fence: lease_b.fence,
expected_version: lease_b.record.version,
attempt_id: "attempt-b".to_owned(),
input_fingerprint: lease_b.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await
.expect("checkpoint B");
store
.checkpoint_dispatch(BeginAttempt {
task_id: lease_a.record.task_id,
owner: "worker_dead".to_owned(),
fence: lease_a.fence,
expected_version: lease_a.record.version,
attempt_id: "attempt-a".to_owned(),
input_fingerprint: lease_a.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: true,
})
.await
.expect("checkpoint A");
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
let recovered = store.claim_batch("worker_live", 4, 10_000).await.unwrap();
let lease_a2 = recovered
.iter()
.find(|l| l.record.task_id == created_a.task_id)
.cloned()
.expect("A recovered");
assert!(lease_a2.fence > lease_a.fence, "recovery bumps fence");
assert!(
matches!(
lease_a2.record.state,
TaskState::Working | TaskState::Queued
),
"A state working/queued"
);
assert!(
!recovered
.iter()
.any(|l| l.record.task_id == created_b.task_id),
"B must not be redispatched"
);
let record_b = store.read(&access(AGENT), created_b.task_id).await.unwrap();
assert_eq!(record_b.state, TaskState::OutcomeUnknown);
// A prior attempt resolved as uncertain.
let attempts_a = store
.attempts(&access(AGENT), created_a.task_id)
.await
.unwrap();
assert!(!attempts_a.is_empty());
assert_eq!(
attempts_a[0].outcome,
Some(AttemptOutcomeClass::OutcomeUnknown)
);
// Further scans do not hand B back.
assert!(
store
.claim_batch("worker_live", 4, 10_000)
.await
.unwrap()
.iter()
.all(|l| l.record.task_id != created_b.task_id)
);
database.cleanup().await;
}
#[tokio::test]
async fn before_dispatch_failed_attempt_does_not_erase_recorded_uncertainty() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let mut req = request(&key("bd_1"), &digest("fbd"), AGENT, 30_000);
req.pins.retry = TaskRetryPolicy::new(2, 0);
let created = store.accept(req).await.unwrap();
let leases = store.claim_batch("worker", 1, 10_000).await.unwrap();
let lease_a = leases.into_iter().next().expect("claim A");
let committed_a = store
.checkpoint_dispatch(BeginAttempt {
task_id: lease_a.record.task_id,
owner: "worker".to_owned(),
fence: lease_a.fence,
expected_version: lease_a.record.version,
attempt_id: "attempt-a".to_owned(),
input_fingerprint: lease_a.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: true,
})
.await
.expect("checkpoint A");
// Attempt A resolves to a bounded retry (uncertain outcome).
let retry = store
.resolve_attempt(
&committed_a,
AttemptOutcome::Uncertain {
error: TaskEnvelope::try_redacted(
&json!({"code":"outcome_unknown","phase":"dispatched"}),
)
.unwrap(),
repeatable: true,
phase: AttemptPhase::Dispatched,
},
)
.await
.expect("resolve A");
let retry_task = match retry {
AttemptResolution::RetryAt { task, .. } => task,
other => panic!("expected RetryAt, got {other:?}"),
};
assert_eq!(retry_task.state, TaskState::Working);
assert!(retry_task.current_attempt_id.is_none());
let retry_version = retry_task.version;
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
// The Task keeps its lease across the pinned backoff: the next attempt checkpoints under the
// same owner/fence and the version the RetryAt record reported.
let committed_b_result = store
.checkpoint_dispatch(BeginAttempt {
task_id: retry_task.task_id,
owner: "worker".to_owned(),
fence: lease_a.fence,
expected_version: retry_version,
attempt_id: "attempt-b".to_owned(),
input_fingerprint: retry_task.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await;
let committed_b = committed_b_result.expect("checkpoint B");
// Attempt B never reached upstream: a before_dispatch failure must NOT overwrite
// the recorded uncertainty of the previously dispatched attempt.
let resolution = store
.resolve_attempt(
&committed_b,
AttemptOutcome::Failed {
error: TaskEnvelope::try_redacted(
&json!({"code":"shutdown_before_dispatch","phase":"before_dispatch"}),
)
.unwrap(),
retryable: false,
phase: AttemptPhase::BeforeDispatch,
},
)
.await
.expect("resolve B");
match resolution {
AttemptResolution::Unknown(record) => assert_eq!(record.state, TaskState::OutcomeUnknown),
other => panic!("expected Unknown, got {other:?}"),
}
let record = store.read(&access(AGENT), created.task_id).await.unwrap();
assert_eq!(record.state, TaskState::OutcomeUnknown);
assert!(record.result.is_none());
let error = record.error.expect("error envelope present");
assert_eq!(error.value()["code"], json!("shutdown_before_dispatch"));
assert_eq!(error.value()["phase"], json!("dispatched"));
assert_eq!(record.attempts, 2);
let attempts = store
.attempts(&access(AGENT), created.task_id)
.await
.unwrap();
assert_eq!(attempts.len(), 2);
assert_eq!(
attempts[0].outcome,
Some(AttemptOutcomeClass::RetryScheduled)
);
assert_eq!(
attempts[0].certainty,
Some(crank_core::AttemptCertainty::Unknown)
);
assert_eq!(attempts[0].phase, Some(AttemptPhase::Dispatched));
assert_eq!(attempts[1].outcome, Some(AttemptOutcomeClass::Failed));
assert_eq!(
attempts[1].certainty,
Some(crank_core::AttemptCertainty::Known)
);
assert_eq!(attempts[1].phase, Some(AttemptPhase::BeforeDispatch));
// Repeated read/cancel must not turn the state terminal.
let again = store.read(&access(AGENT), created.task_id).await.unwrap();
assert_eq!(again.state, TaskState::OutcomeUnknown);
database.cleanup().await;
}
#[tokio::test]
async fn before_dispatch_failed_single_attempt_stays_failed() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let created = store
.accept(request(&key("bd_single"), &digest("fbf"), AGENT, 30_000))
.await
.unwrap();
let leases = store.claim_batch("worker", 1, 10_000).await.unwrap();
let lease = leases.into_iter().next().expect("claim");
let committed = store
.checkpoint_dispatch(BeginAttempt {
task_id: lease.record.task_id,
owner: "worker".to_owned(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: "attempt-single".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await
.expect("checkpoint");
let resolution = store
.resolve_attempt(
&committed,
AttemptOutcome::Failed {
error: TaskEnvelope::try_redacted(
&json!({"code":"shutdown_before_dispatch","phase":"before_dispatch"}),
)
.unwrap(),
retryable: false,
phase: AttemptPhase::BeforeDispatch,
},
)
.await
.expect("resolve");
match resolution {
AttemptResolution::Finished(record) => assert_eq!(record.state, TaskState::Failed),
other => panic!("expected Finished, got {other:?}"),
}
let record = store.read(&access(AGENT), created.task_id).await.unwrap();
assert_eq!(record.state, TaskState::Failed);
let error = record.error.expect("error envelope present");
assert_eq!(error.value()["phase"], json!("before_dispatch"));
database.cleanup().await;
}
@@ -0,0 +1,167 @@
//! Story 4.3 ATDD: durable Task leases, fencing, crash recovery and bounded claim behavior.
//!
//! Split from `tasks.rs` to keep both integration files inside the code-health limit.
use super::*;
use serde_json::json;
/// Story 4.3 ATDD: a worker crash before the durable dispatch checkpoint leaves only an expired
/// lease. PostgreSQL is authoritative, so another worker recovers the Task and the dead owner can
/// never commit a terminal outcome with its old fence.
#[tokio::test]
async fn task_worker_expired_pre_checkpoint_lease_is_recovered_and_stale_owner_cannot_commit() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let created = store
.accept(request(&key("recover1"), &digest("f2"), AGENT, 30_000))
.await
.unwrap();
let first = store.claim_batch("worker_dead", 1, 150).await.unwrap();
assert_eq!(first.len(), 1);
assert_eq!(first[0].record.task_id, created.task_id);
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
// Recovery on the next bounded authoritative scan, with no Valkey wake-up.
let recovered = store.claim_batch("worker_live", 4, 10_000).await.unwrap();
let lease = recovered
.into_iter()
.find(|lease| lease.record.task_id == created.task_id)
.expect("expired pre-checkpoint Task is recovered");
assert!(lease.fence > first[0].fence, "recovery bumps the fence");
// The previous owner's terminal CAS is rejected; the live owner wins exactly once.
let stale = store
.finish_leased(
TaskFinishRequest {
access: access(AGENT),
task_id: created.task_id,
expected_state: first[0].record.state,
expected_version: first[0].record.version,
target_state: TaskState::Completed,
result: Some(TaskEnvelope::try_redacted(&json!({"stale": true})).unwrap()),
error: None,
},
"worker_dead",
first[0].fence,
)
.await;
assert!(matches!(stale, Err(TaskError::StateConflict)));
let finished = store
.finish_leased(
TaskFinishRequest {
access: access(AGENT),
task_id: created.task_id,
expected_state: lease.record.state,
expected_version: lease.record.version,
target_state: TaskState::Completed,
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
error: None,
},
"worker_live",
lease.fence,
)
.await
.expect("current lease wins the terminal CAS");
assert_eq!(finished.state, TaskState::Completed);
database.cleanup().await;
}
/// Story 4.3 ATDD: after the dispatch checkpoint the outbound outcome is uncertain. An expired
/// attempt resolves to `outcome_unknown`, is never redispatched automatically and releases the
/// admission slot it held.
#[tokio::test]
async fn task_worker_expired_checkpointed_attempt_becomes_outcome_unknown_without_redispatch() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let created = store
.accept(request(&key("uncertain1"), &digest("f3"), AGENT, 30_000))
.await
.unwrap();
let leases = store.claim_batch("worker_dead", 1, 150).await.unwrap();
let lease = leases.into_iter().next().expect("claim");
store
.checkpoint_dispatch(BeginAttempt {
task_id: lease.record.task_id,
owner: "worker_dead".to_owned(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: "attempt-uncertain".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await
.expect("durable dispatch checkpoint");
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
// The authority scan resolves uncertainty instead of handing the attempt to another worker.
let after = store.claim_batch("worker_live", 4, 10_000).await.unwrap();
assert!(
after.is_empty(),
"a checkpointed attempt must never be redispatched"
);
let record = store.read(&access(AGENT), created.task_id).await.unwrap();
assert_eq!(record.state, TaskState::OutcomeUnknown);
assert_eq!(
record.current_attempt_id.as_deref(),
Some("attempt-uncertain")
);
let intents: i64 = sqlx::query_scalar(
"select count(*)::bigint from execution_admission_entries where task_id = $1",
)
.bind(created.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(intents, 0, "uncertain terminal outcome releases capacity");
// Still no automatic retry on the following scans.
assert!(
store
.claim_batch("worker_live", 4, 10_000)
.await
.unwrap()
.is_empty()
);
database.cleanup().await;
}
/// Story 4.3 ATDD: concurrent claimers never hold the same Task at once.
#[tokio::test]
async fn task_worker_concurrent_claimers_never_double_lease_one_task() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = std::sync::Arc::new(store_with(registry.pool().clone(), 8));
let created = store
.accept(request(&key("concurrent1"), &digest("f4"), AGENT, 30_000))
.await
.unwrap();
let mut handles = Vec::new();
for index in 0..4 {
let store = store.clone();
handles.push(tokio::spawn(async move {
store
.claim_batch(&format!("worker_{index}"), 1, 10_000)
.await
.unwrap()
}));
}
let mut leased = Vec::new();
for handle in handles {
leased.extend(handle.await.unwrap());
}
assert_eq!(leased.len(), 1, "exactly one owner wins the claim");
assert_eq!(leased[0].record.task_id, created.task_id);
database.cleanup().await;
}
+389 -13
View File
@@ -22,9 +22,15 @@ use serde_json::{Map, Value, json};
use time::OffsetDateTime;
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
use tracing::{Instrument, Span, debug, warn};
use uuid::Uuid;
use crate::task_attempt::{
PrepareAttempt, TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
TaskAttemptRefusal,
};
use crate::task_worker::TaskDispatchPermit;
use crate::{
AdapterResponse, ExecutionAuthorization, PreparedRequest, RuntimeError,
AdapterResponse, ExecutionAuthorization, ExecutionRequestError, PreparedRequest, RuntimeError,
RuntimeExecutionRequest, RuntimeLimits, RuntimeOperation, RuntimeRequestContext,
auth::apply_resolved_auth,
request_preparation::{adapter_prepared_request, validate_selected_body_variant},
@@ -34,6 +40,18 @@ use crate::{
},
};
/// How many outbound sends one prepared request may perform.
///
/// `Legacy` keeps the existing synchronous `tools/call` retry behaviour inside the transport loop.
/// `Single` performs exactly one send, because a durable Task attempt already has its own committed
/// checkpoint and resolution around every send; it carries the pinned attempt ordinal and budget so
/// the one `adapter_attempt` span still names the real durable attempt.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum SendMode {
Legacy,
Single { attempt: u32, max_attempts: u32 },
}
#[derive(Clone)]
pub struct RuntimeExecutor {
adapters: AdapterRegistry,
@@ -178,8 +196,305 @@ impl RuntimeExecutor {
pub(crate) async fn execute_request(
&self,
request: RuntimeExecutionRequest<'_>,
) -> Result<(Value, Value), RuntimeError> {
self.execute_request_impl(request, None, SendMode::Legacy)
.await
}
/// Drives the bounded, durable retry budget of one leased background Task (Story 4.4).
///
/// Each outbound send is preceded by its own committed attempt checkpoint and followed by one
/// atomic resolution; the pinned budget, the safe-retry decision and the accepted absolute
/// deadline are enforced by PostgreSQL, so a restart or a second worker cannot extend them.
pub async fn execute_task_outcome(
&self,
context: &TaskAttemptContext,
authority: &dyn TaskAttemptAuthority,
) -> TaskAttemptOutcome {
let store = context.store().clone();
let lease = context.lease().clone();
let mut record = lease.record.clone();
loop {
if context.is_cancelled() || context.is_stopping() {
return TaskAttemptOutcome::Stopped;
}
let current_lease = crank_core::TaskLease {
record: record.clone(),
owner: lease.owner.clone(),
fence: lease.fence,
lease_until: lease.lease_until,
};
let plan = match authority.prepare_attempt(&current_lease).await {
Ok(PrepareAttempt::Ready(plan)) => *plan,
Ok(PrepareAttempt::InputRequired(request)) => {
// A missing required input is suspended before the dispatch checkpoint, so this
// generation provably produced no outbound side effect. A CAS/store failure stops
// without writing anything new; the lease expiry recovery makes it retryable.
return match store.suspend_for_input(&current_lease, request).await {
Ok(_) => TaskAttemptOutcome::WaitingForInput,
Err(crank_core::TaskError::StateConflict)
| Err(crank_core::TaskError::NotFound)
| Err(crank_core::TaskError::FingerprintConflict)
| Err(crank_core::TaskError::InputExpired) => TaskAttemptOutcome::Stopped,
Err(error) => {
warn!(error = %error, "input suspension failed; no outbound call");
TaskAttemptOutcome::Stopped
}
};
}
Err(TaskAttemptRefusal::Stopped) => return TaskAttemptOutcome::Stopped,
Err(TaskAttemptRefusal::Failed { code }) => {
// A refused attempt never erases a proven ambiguity: if an earlier attempt of this
// Task may already have reached the upstream, the store keeps the Task in the
// non-terminal `outcome_unknown` instead of downgrading it to a plain failure.
return match store.refuse_attempt(&current_lease, &code).await {
Ok(record) if record.state == crank_core::TaskState::OutcomeUnknown => {
TaskAttemptOutcome::OutcomeUnknown
}
_ => TaskAttemptOutcome::Failed { code },
};
}
};
let attempt_id = format!("attempt-{}", Uuid::now_v7());
let begin = crank_core::BeginAttempt {
task_id: record.task_id,
owner: lease.owner.clone(),
fence: lease.fence,
expected_version: record.version,
attempt_id: attempt_id.clone(),
input_fingerprint: record.effective_input_fingerprint.clone(),
upstream_key_digest: plan.upstream_key_digest.clone(),
retry_allowed: plan.retry_allowed,
};
let attempt = match store.checkpoint_dispatch(begin).await {
Ok(attempt) => attempt,
// Cancellation, lease loss or a concurrent lifecycle change: nothing was dispatched.
Err(crank_core::TaskError::StateConflict)
| Err(crank_core::TaskError::NotFound)
| Err(crank_core::TaskError::FingerprintConflict) => {
return TaskAttemptOutcome::Stopped;
}
// An unproven commit is never retried by guess: no outbound call without a proven
// durable checkpoint.
Err(error) => {
warn!(error = %error, "durable dispatch checkpoint failed; no outbound call");
return TaskAttemptOutcome::Stopped;
}
};
if context.is_stopping() {
let code = "shutdown_before_dispatch".to_owned();
// A stop that lands before the send is a proven failure of *this* attempt only: when
// an earlier attempt of the same Task already lost its confirmation, the store keeps
// the non-terminal `outcome_unknown` instead of downgrading it to a plain failure.
return match store
.resolve_attempt(
&attempt,
crank_core::AttemptOutcome::Failed {
error: bounded_attempt_error(
&code,
crank_core::AttemptPhase::BeforeDispatch,
),
retryable: false,
phase: crank_core::AttemptPhase::BeforeDispatch,
},
)
.await
{
Ok(crank_core::AttemptResolution::Unknown(_)) => {
TaskAttemptOutcome::OutcomeUnknown
}
_ => TaskAttemptOutcome::Failed { code },
};
}
if context.is_cancelled() {
let _ = store
.resolve_attempt(
&attempt,
crank_core::AttemptOutcome::Uncertain {
error: bounded_attempt_error(
"outcome_unknown",
crank_core::AttemptPhase::BeforeDispatch,
),
repeatable: false,
phase: crank_core::AttemptPhase::BeforeDispatch,
},
)
.await;
return TaskAttemptOutcome::OutcomeUnknown;
}
// Each attempt owns exactly one `adapter_attempt` span, created around the single send
// inside the transport loop so the accepted trace shape (durable attempt → runtime
// execution → adapter attempt) is preserved.
let result = self.execute_task_send(&plan, &attempt).await;
let (outcome, code) = self.classify_task_attempt(&plan, result);
match store.resolve_attempt(&attempt, outcome).await {
Ok(crank_core::AttemptResolution::Finished(task)) => {
return if task.state == crank_core::TaskState::Completed {
TaskAttemptOutcome::Completed
} else {
TaskAttemptOutcome::Failed {
code: code.unwrap_or_else(|| "task_failed".to_owned()),
}
};
}
Ok(crank_core::AttemptResolution::Unknown(_)) => {
return TaskAttemptOutcome::OutcomeUnknown;
}
Ok(crank_core::AttemptResolution::RetryAt {
task,
next_attempt_at_unix_ms,
}) => {
record = task;
let remaining =
next_attempt_at_unix_ms.saturating_sub(crank_core::current_unix_ms());
let delay = Duration::from_millis(u64::try_from(remaining).unwrap_or(0));
if !self.wait_before_attempt(context, delay).await {
return TaskAttemptOutcome::Stopped;
}
}
// A stale fence or version lost the resolution CAS: another writer owns the Task now.
Err(_) => return TaskAttemptOutcome::Stopped,
}
}
}
/// One outbound send of one committed durable attempt.
async fn execute_task_send(
&self,
plan: &TaskAttemptPlan,
attempt: &crank_core::AttemptLease,
) -> Result<Value, RuntimeError> {
let permit = TaskDispatchPermit::new(
attempt.attempt.task_id,
plan.operation.operation_id.as_str().to_owned(),
plan.operation.operation_version,
plan.workspace_id.as_str().to_owned(),
plan.agent_id.as_ref().map(|id| id.as_str().to_owned()),
attempt.attempt.owner.clone(),
attempt.attempt.fence,
attempt.attempt.attempt_id.clone(),
);
let request = match RuntimeExecutionRequest::try_new(
&plan.workspace_id,
plan.origin,
plan.agent_id.as_ref(),
&plan.operation,
&plan.input,
plan.authorization,
plan.resolved_auth.as_ref(),
&plan.request_context,
plan.deadline,
) {
Ok(request) => request,
Err(ExecutionRequestError::DeadlineElapsed) => {
return Err(RuntimeError::ExecutionDeadlineElapsed {
may_have_dispatched: false,
});
}
Err(_) => return Err(RuntimeError::InvalidExecutionRequest),
};
let (output, _preview) = self
.execute_request_impl(
request,
Some(permit),
SendMode::Single {
attempt: attempt.attempt.ordinal,
max_attempts: attempt.task.pins.retry.max_attempts,
},
)
.await?;
Ok(output)
}
/// Maps one outbound result of a durable attempt onto its persisted attempt outcome.
fn classify_task_attempt(
&self,
plan: &TaskAttemptPlan,
result: Result<Value, RuntimeError>,
) -> (crank_core::AttemptOutcome, Option<String>) {
match result {
Ok(output) => match crank_core::TaskEnvelope::try_redacted(&output) {
Ok(envelope) => (crank_core::AttemptOutcome::Completed(envelope), None),
Err(_) => {
let code = "result_not_redactable".to_owned();
(
crank_core::AttemptOutcome::Failed {
error: bounded_attempt_error(
&code,
crank_core::AttemptPhase::Dispatched,
),
retryable: false,
phase: crank_core::AttemptPhase::Dispatched,
},
Some(code),
)
}
},
Err(error) => {
let phase = if runtime_error_may_have_dispatched(&error) {
crank_core::AttemptPhase::Dispatched
} else {
crank_core::AttemptPhase::BeforeDispatch
};
let correlation = crank_core::CorrelationContext::new(
plan.request_context.request_id.clone(),
plan.request_context.trace_context.clone(),
);
let failure = crate::normalize_runtime_error_for_operation(
&error,
&correlation,
&plan.operation,
);
let code = format!("{:?}", failure.error_code());
let safe_to_repeat = retry_is_allowed(&plan.operation, &error, plan.retry_allowed);
let envelope = bounded_attempt_error(&code, phase);
let outcome = match failure.outcome_certainty() {
crank_core::OutcomeCertainty::OutcomeUnknown => {
crank_core::AttemptOutcome::Uncertain {
error: envelope,
repeatable: safe_to_repeat,
phase,
}
}
_ => crank_core::AttemptOutcome::Failed {
error: envelope,
retryable: safe_to_repeat,
phase,
},
};
(outcome, Some(code))
}
}
}
/// Bounded cooperative wait inside the pinned backoff window. The lease keeps being renewed by
/// the worker, so a durable cancellation or lease loss still stops the local work here.
async fn wait_before_attempt(&self, context: &TaskAttemptContext, delay: Duration) -> bool {
let deadline = Instant::now() + delay;
loop {
if context.is_cancelled() || context.is_stopping() {
return false;
}
let now = Instant::now();
if now >= deadline {
return true;
}
tokio::time::sleep((deadline - now).min(Duration::from_millis(50))).await;
}
}
pub(crate) async fn execute_request_impl(
&self,
request: RuntimeExecutionRequest<'_>,
permit: Option<TaskDispatchPermit>,
send_mode: SendMode,
) -> Result<(Value, Value), RuntimeError> {
let operation = request.operation();
// Derive trusted dispatch from a present permit; a missing permit always uses ordinary admission.
let trusted_task_dispatch = permit.is_some();
// The durable Attempt ID written by the dispatch checkpoint travels with the permit, so the
// trace names the real persisted attempt instead of inventing one (Story 4.3 AC).
let durable_attempt_id = permit.as_ref().map(|permit| permit.attempt_id().to_owned());
let input = request.input();
let request_context = request.request_context();
debug_assert_eq!(request.authorization(), ExecutionAuthorization::Authorized);
@@ -205,7 +520,9 @@ impl RuntimeExecutor {
let dispatch_started = Arc::new(AtomicBool::new(false));
// True until the admission gate answered. A deadline that fires while the request is still
// waiting for a permit is an admission refusal with zero dispatch, never an upstream timeout.
let admission_pending = Arc::new(AtomicBool::new(self.admission.is_some()));
let admission_pending = Arc::new(AtomicBool::new(
!trusted_task_dispatch && self.admission.is_some(),
));
let invocation_metrics = ToolInvocationMetrics::start_with_exemplar(
metric_invocation_source(Some(request_context)),
request_context
@@ -222,14 +539,19 @@ impl RuntimeExecutor {
// One admission gate for every executable path (Story 4.1): the published policy decides
// the effective mode, the absolute deadline includes queue waiting and the scoped permit
// is held for the whole local execution future, including retries.
let mut lease = crate::admission::acquire(
self.admission.as_ref(),
self.coordination_store.as_deref(),
&request,
execution_deadline,
)
.await?;
admission_pending.store(false, Ordering::Release);
let mut lease = if trusted_task_dispatch {
None
} else {
let result = crate::admission::acquire(
self.admission.as_ref(),
self.coordination_store.as_deref(),
&request,
execution_deadline,
)
.await?;
admission_pending.store(false, Ordering::Release);
result
};
let _permit = self.acquire_unary_permit(operation)?;
let _inflight = InFlightGuard::runtime();
let mapping_span = Stage::RuntimeArgumentsMap.span();
@@ -246,6 +568,8 @@ impl RuntimeExecutor {
prepared_request,
Some(request_context),
Arc::clone(&dispatch_started),
durable_attempt_id.as_deref(),
send_mode,
)
.await;
if let Some(lease) = lease.as_mut() {
@@ -341,6 +665,7 @@ impl RuntimeExecutor {
Ok(prepared)
}
#[allow(clippy::too_many_arguments)]
async fn execute_prepared(
&self,
operation: &RuntimeOperation,
@@ -348,6 +673,8 @@ impl RuntimeExecutor {
prepared_request: PreparedRequest,
request_context: Option<&RuntimeRequestContext>,
dispatch_started: Arc<AtomicBool>,
durable_attempt_id: Option<&str>,
send_mode: SendMode,
) -> Result<Value, RuntimeError> {
let mut prepared_request = prepared_request;
let idempotency_applicable = crate::idempotency::policy(operation).is_some();
@@ -397,7 +724,11 @@ impl RuntimeExecutor {
approval_result?;
}
let idempotency = if idempotency_applicable {
// The synchronous coordination guard is the replay authority of one `tools/call`. A durable
// Task attempt already has a PostgreSQL attempt authority around every send, so the guard is
// skipped there: an ambiguous first response must not block the bounded durable retry. The
// stable idempotency key is still delivered to the upstream by `prepare_idempotency` above.
let idempotency = if idempotency_applicable && matches!(send_mode, SendMode::Legacy) {
let idempotency_span = Stage::RuntimeIdempotency.span();
let result = crate::idempotency::begin(
self.coordination_store.as_deref(),
@@ -464,6 +795,8 @@ impl RuntimeExecutor {
operation,
idempotency_key.as_deref(),
),
durable_attempt_id,
send_mode,
)
.await;
if let Ok(response) = &adapter_response {
@@ -597,6 +930,7 @@ impl RuntimeExecutor {
.map_err(|error| map_protocol_adapter_error(operation, error))
}
#[allow(clippy::too_many_arguments)]
async fn execute_adapter_with_retry(
&self,
operation: &RuntimeOperation,
@@ -604,10 +938,35 @@ impl RuntimeExecutor {
request_context: Option<&RuntimeRequestContext>,
dispatch_started: Arc<AtomicBool>,
verified_idempotency: bool,
durable_attempt_id: Option<&str>,
send_mode: SendMode,
) -> Result<AdapterResponse, RuntimeError> {
let max_attempts = retry_attempts(operation);
let mut attempt = 1;
// `max_attempts` bounds the transport loop; `reported_max_attempts` is the pinned Task budget
// the attempt span names. A durable attempt performs exactly one send, so its transport loop
// always has a single iteration even when the Task may retry later.
let (max_attempts, first_attempt, reported_max_attempts) = match send_mode {
SendMode::Single {
attempt,
max_attempts,
} => (1, attempt, max_attempts),
SendMode::Legacy => {
let attempts = retry_attempts(operation);
(attempts, 1, attempts)
}
};
let mut attempt = first_attempt;
loop {
// Task dispatches reuse the durable Attempt ID committed by the dispatch checkpoint. Only
// paths without a durable attempt (synchronous tools/call) mint a synthetic identity.
let attempt_id = durable_attempt_id
.map(str::to_owned)
.unwrap_or_else(|| format!("attempt-{}", Uuid::now_v7()));
let attempt_span = tracing::info_span!(
"adapter_attempt",
attempt,
max_attempts = reported_max_attempts,
attempt_id = %attempt_id
);
let result = self
.execute_adapter(
operation,
@@ -615,6 +974,7 @@ impl RuntimeExecutor {
request_context,
Arc::clone(&dispatch_started),
)
.instrument(attempt_span)
.await;
match result {
Ok(response) => return Ok(response),
@@ -756,6 +1116,22 @@ fn sanitized_request_preview(prepared: &PreparedRequest) -> Value {
})
}
/// Bounded persisted error envelope of one durable attempt: a closed code and the attempt phase.
/// It never carries an upstream body, status text, URL or credential.
fn bounded_attempt_error(code: &str, phase: crank_core::AttemptPhase) -> crank_core::TaskEnvelope {
crank_core::TaskEnvelope::try_redacted(&json!({
"code": code,
"phase": phase.as_str(),
}))
.unwrap_or_else(|_| {
crank_core::TaskEnvelope::try_redacted(&json!({
"code": "task_error_not_recordable",
"phase": phase.as_str(),
}))
.expect("bounded attempt error envelope is always valid")
})
}
fn runtime_error_may_have_dispatched(error: &RuntimeError) -> bool {
matches!(
error,
+9
View File
@@ -15,6 +15,8 @@ mod rate_limit;
mod request_context;
mod request_preparation;
mod response_cache;
mod task_attempt;
mod task_worker;
mod tasks;
pub fn validate_resource_template_parameter(
schema: &crank_schema::Schema,
@@ -115,4 +117,11 @@ pub use rate_limit::{
};
pub use request_context::{MeteringContext, ResponseCacheScope, RuntimeRequestContext};
pub use secret_crypto::SecretCrypto;
pub use task_attempt::{
PrepareAttempt, TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
TaskAttemptRefusal, missing_required_input,
};
pub use task_worker::{
TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
};
pub use tasks::{TaskAcceptDraft, TaskOutcome, TaskService};
+140
View File
@@ -0,0 +1,140 @@
//! Durable per-outbound attempt execution of one background Task (Story 4.4).
//!
//! The authoritative state of a Task stays in PostgreSQL. This module only carries the owned
//! execution plan of one attempt, the durable control (store + lease + cooperative stop) and the
//! bounded seam the application layer implements to revalidate authority before each attempt.
//! The single retry loop lives in [`crate::RuntimeExecutor::execute_task_outcome`]; there is no second
//! loop in the worker and no retry inside the transport adapter.
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::time::Instant;
use async_trait::async_trait;
use crank_core::{AgentId, ExecutionOrigin, TaskLease, TaskStore, WorkspaceId};
use crate::{ExecutionAuthorization, ResolvedAuth, RuntimeOperation, RuntimeRequestContext};
/// Owned execution plan of one durable attempt.
///
/// It is rebuilt by the application through [`TaskAttemptAuthority::prepare_attempt`] before every
/// attempt, so a republished catalog, a revoked key or an elapsed deadline is revalidated between two
/// attempts instead of being trusted from the first one.
pub struct TaskAttemptPlan {
pub workspace_id: WorkspaceId,
pub origin: ExecutionOrigin,
pub agent_id: Option<AgentId>,
pub operation: RuntimeOperation,
pub input: serde_json::Value,
pub authorization: ExecutionAuthorization,
pub resolved_auth: Option<ResolvedAuth>,
pub request_context: RuntimeRequestContext,
/// Absolute deadline pinned at acceptance; never re-derived from a later publication.
pub deadline: Instant,
/// Safe-to-retry decision of this attempt: a read, or a write whose upstream idempotency is
/// backed by server-owned evidence. A client-supplied key or header never makes this true.
pub retry_allowed: bool,
/// Digest of the stable upstream idempotency key (never the raw key or a header value).
pub upstream_key_digest: Option<String>,
}
/// Application-owned seam: revalidate authority and prepare exactly one durable attempt.
#[async_trait]
pub trait TaskAttemptAuthority: Send + Sync {
async fn prepare_attempt(
&self,
lease: &TaskLease,
) -> Result<PrepareAttempt, TaskAttemptRefusal>;
}
/// Typed preparation of one attempt: either it is ready to be checkpointed and dispatched, or the
/// pinned operation still lacks a required input and the Task must be suspended before any side
/// effect. The two are explicit so a real authority failure is never turned into a user question.
pub enum PrepareAttempt {
Ready(Box<TaskAttemptPlan>),
InputRequired(crank_core::TaskInputRequest),
}
/// Path of a required item the pinned schema still misses before dispatch.
///
/// This is the runtime-owned decision the application calls before checkpointing an attempt: only a
/// typed missing required field becomes a user question, every other schema violation keeps the
/// existing pre-dispatch attempt validation. Exposed as a plain value so the application layer never
/// maps execution validation itself (execution boundary).
pub fn missing_required_input(
schema: &crank_schema::Schema,
input: &serde_json::Value,
) -> Option<String> {
match schema.validate_shape(input) {
Err(crank_schema::SchemaError::MissingField { path }) => Some(path),
_ => None,
}
}
/// Typed refusal of one attempt preparation.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum TaskAttemptRefusal {
/// Proven refusal before any dispatch of this attempt; the bounded code is persisted.
Failed { code: String },
/// Durable control changed (cancellation, lease loss, shutdown): write nothing new from here.
Stopped,
}
/// Durable control of one leased Task: the store, the leased record and cooperative stop signals.
#[derive(Clone)]
pub struct TaskAttemptContext {
pub(crate) store: Arc<dyn TaskStore>,
pub(crate) lease: TaskLease,
pub(crate) cancelled: Arc<AtomicBool>,
pub(crate) stopping: Arc<AtomicBool>,
}
impl TaskAttemptContext {
pub(crate) fn new(
store: Arc<dyn TaskStore>,
lease: TaskLease,
cancelled: Arc<AtomicBool>,
stopping: Arc<AtomicBool>,
) -> Self {
Self {
store,
lease,
cancelled,
stopping,
}
}
pub fn store(&self) -> &Arc<dyn TaskStore> {
&self.store
}
pub fn lease(&self) -> &TaskLease {
&self.lease
}
/// Cooperative stop: the lease renewal or an observed durable cancellation set this flag.
pub fn is_cancelled(&self) -> bool {
self.cancelled.load(std::sync::atomic::Ordering::Acquire)
}
/// Process shutdown was requested: no new outbound send may start after the current checkpoint.
pub fn is_stopping(&self) -> bool {
self.stopping.load(std::sync::atomic::Ordering::Acquire)
}
}
/// Result of driving the whole pinned retry budget of one leased Task.
#[derive(Clone, Debug, PartialEq)]
pub enum TaskAttemptOutcome {
/// The Task committed `completed` (exactly one terminal write).
Completed,
/// The Task committed `failed` without dispatch or with a proven failure.
Failed { code: String },
/// The Task is non-terminal `outcome_unknown`; it is never retried or cancelled generically.
OutcomeUnknown,
/// The Task was suspended as `input_required` before its first dispatch: no side effect happened
/// and the worker future must stop; the answer is applied through the application entry.
WaitingForInput,
/// Nothing new was written by this worker: cancellation, lease loss or an unavailable store.
Stopped,
}
+433
View File
@@ -0,0 +1,433 @@
//! Fenced durable Task worker loop for Story 4.3.
//!
//! The worker is intentionally scheduler-ish: PostgreSQL is authoritative, polling is bounded,
//! Valkey may only wake it up (never represented here as correctness), and every lease-protected
//! write goes through CAS using owner + fencing token. The actual executable input and revalidation
//! policy arrive via [`TaskHandler`] implemented by the application layer; `TaskRecord.input` is a
//! redacted projection and is never dispatched.
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use async_trait::async_trait;
use crank_core::{TaskError, TaskLease, TaskStore};
use tokio::sync::watch;
use tokio::task::JoinSet;
use tracing::{Instrument, debug, warn};
use uuid::Uuid;
use crate::task_attempt::{TaskAttemptContext, TaskAttemptOutcome};
/// Configuration for [`TaskWorker`]. Durations are capped by the caller to story timing budgets.
#[derive(Clone, Debug)]
pub struct TaskWorkerConfig {
/// Stable random-ish owner identity for this process instance. Required lease owner.
pub owner: String,
/// Maximum number of Tasks claimed in one bounded PG scan.
pub batch_limit: u32,
/// Lease length in milliseconds. Recovery stays within 60s when <= 30s is used.
pub lease_ms: i64,
/// Authoritative PG polling interval when no wake-up hint is available.
pub poll_interval: Duration,
/// Renewal cadence while processing a lease.
pub renewal_interval: Duration,
/// Concurrent in-flight Tasks for this worker.
pub max_inflight: usize,
/// Maximum time to wait for active work during graceful shutdown.
pub shutdown_timeout: Duration,
}
impl Default for TaskWorkerConfig {
fn default() -> Self {
Self {
owner: format!("task-worker-{}", Uuid::now_v7()),
batch_limit: 8,
lease_ms: 30_000,
poll_interval: Duration::from_millis(250),
renewal_interval: Duration::from_millis(5_000),
max_inflight: 4,
shutdown_timeout: Duration::from_secs(30),
}
}
}
/// Application-owned seam for pre-dispatch revalidation and execution.
///
/// The worker owns lease renewal/CAS/fencing and passes a durable [`TaskAttemptContext`]. The
/// application implements [`crate::TaskAttemptAuthority`] and drives the pinned retry budget through
/// [`crate::RuntimeExecutor::execute_task_outcome`]; it must not trust `TaskRecord.input` as
/// executable payload.
#[async_trait]
pub trait TaskHandler: Send + Sync {
async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError>;
async fn execute(&self, context: TaskAttemptContext) -> TaskAttemptOutcome;
}
/// Sealed, single-use trusted dispatch permit created by [`TaskWorker`] after a durable lease and
/// successful `checkpoint_dispatch`. It is neither `Clone` nor constructible outside this crate, and
/// it binds one durable Task attempt to its execution request.
pub struct TaskDispatchPermit {
task_id: crank_core::TaskId,
operation_id: String,
operation_version: u32,
workspace_id: String,
agent_id: Option<String>,
owner: String,
fence: u64,
attempt_id: String,
consumed: std::sync::atomic::AtomicBool,
}
impl TaskDispatchPermit {
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
task_id: crank_core::TaskId,
operation_id: String,
operation_version: u32,
workspace_id: String,
agent_id: Option<String>,
owner: String,
fence: u64,
attempt_id: String,
) -> Self {
Self {
task_id,
operation_id,
operation_version,
workspace_id,
agent_id,
owner,
fence,
attempt_id,
consumed: std::sync::atomic::AtomicBool::new(false),
}
}
pub fn task_id(&self) -> crank_core::TaskId {
self.task_id
}
pub fn operation_id(&self) -> &str {
&self.operation_id
}
pub fn operation_version(&self) -> u32 {
self.operation_version
}
pub fn workspace_id(&self) -> &str {
&self.workspace_id
}
pub fn agent_id(&self) -> Option<&str> {
self.agent_id.as_deref()
}
pub fn owner(&self) -> &str {
&self.owner
}
pub fn fence(&self) -> u64 {
self.fence
}
pub fn attempt_id(&self) -> &str {
&self.attempt_id
}
pub fn is_consumed(&self) -> bool {
self.consumed.load(std::sync::atomic::Ordering::Acquire)
}
pub fn consume(&self) -> bool {
!self
.consumed
.swap(true, std::sync::atomic::Ordering::AcqRel)
}
}
#[derive(thiserror::Error, Debug)]
pub enum TaskWorkerError {
#[error("task store error: {0}")]
Store(#[from] TaskError),
#[error("invalid worker configuration: {0}")]
Config(String),
#[error("task terminal transition CAS failed")]
CasRejected,
}
pub struct TaskWorker {
store: Arc<dyn TaskStore>,
handler: Arc<dyn TaskHandler>,
config: TaskWorkerConfig,
stopping: Arc<AtomicBool>,
}
impl TaskWorker {
pub fn new(
store: Arc<dyn TaskStore>,
handler: Arc<dyn TaskHandler>,
config: TaskWorkerConfig,
) -> Result<Self, TaskWorkerError> {
if config.owner.is_empty() {
return Err(TaskWorkerError::Config(
"owner must not be empty".to_owned(),
));
}
if config.batch_limit == 0 || config.batch_limit > 64 || config.lease_ms <= 0 {
return Err(TaskWorkerError::Config(
"batch_limit must be 1..=64 and lease_ms must be positive".to_owned(),
));
}
if config.lease_ms > 30_000
|| config.poll_interval.is_zero()
|| config.poll_interval > Duration::from_secs(30)
|| config.renewal_interval.is_zero()
|| config.renewal_interval >= Duration::from_millis(config.lease_ms as u64)
|| config.max_inflight == 0
|| config.shutdown_timeout.is_zero()
|| config.shutdown_timeout > Duration::from_secs(30)
{
return Err(TaskWorkerError::Config(
"lease, polling, renewal, concurrency, or shutdown exceeds story timing bounds"
.to_owned(),
));
}
Ok(Self {
store,
handler,
config,
stopping: Arc::new(AtomicBool::new(false)),
})
}
/// Signal graceful shutdown. New claims stop; active handlers are cooperatively cancelled.
pub fn stop(&self) {
self.stopping.store(true, Ordering::Release);
}
/// Returns a handle whose drop does not stop the worker; call `stop()` on the worker.
pub fn stop_flag(&self) -> Arc<AtomicBool> {
self.stopping.clone()
}
/// Run until `stop()` is called. Uses PostgreSQL as the authoritative bounded scan.
pub async fn run(&self) -> Result<(), TaskWorkerError> {
self.run_inner(None).await
}
/// Run until either `stop()` or the provided shutdown receiver becomes true.
pub async fn run_until_shutdown(
&self,
mut shutdown: watch::Receiver<bool>,
) -> Result<(), TaskWorkerError> {
self.run_inner(Some(&mut shutdown)).await
}
async fn run_inner(
&self,
mut shutdown: Option<&mut watch::Receiver<bool>>,
) -> Result<(), TaskWorkerError> {
let mut active: JoinSet<()> = JoinSet::new();
loop {
if self.stopping.load(Ordering::Acquire) {
break;
}
if let Some(rx) = shutdown.as_mut()
&& *rx.borrow()
{
break;
}
// Reap finished attempts before claiming more.
while active.try_join_next().is_some() {}
// The bounded TTL sweep runs on every poll, including when all local slots are busy: an
// expired input wait must end `failed` with an audit reason even without free capacity.
match self.store.expire_input_waits(self.config.batch_limit).await {
Ok(expired) if expired > 0 => {
debug!(expired, "expired input waits were failed")
}
Ok(_) => {}
Err(error) => {
warn!(error = %error, "input TTL sweep failed; retrying on next scan")
}
}
let available = self.config.max_inflight.saturating_sub(active.len());
if available > 0 {
let batch_limit = self
.config
.batch_limit
.min(u32::try_from(available).unwrap_or(u32::MAX));
match self
.store
.claim_batch(&self.config.owner, batch_limit, self.config.lease_ms)
.await
{
Ok(leases) => {
for lease in leases {
let store = self.store.clone();
let handler = self.handler.clone();
let owner = self.config.owner.clone();
let lease_ms = self.config.lease_ms;
let renewal_interval = self.config.renewal_interval;
let stopping = self.stopping.clone();
active.spawn(async move {
process_lease(
store,
handler,
owner,
lease,
lease_ms,
renewal_interval,
stopping,
)
.await;
});
}
}
Err(error) => {
warn!(error = %error, "bounded task claim failed; retrying on next scan");
}
}
}
tokio::select! {
_ = tokio::time::sleep(self.config.poll_interval) => {}
_ = async {
if let Some(rx) = shutdown.as_mut() { let _ = rx.changed().await; } else { std::future::pending::<()>().await }
} => {}
}
}
// Graceful shutdown: stop claims (already) and let active work finish/checkpoint, bounded.
// The same stop flag reaches cooperative cancellation and lease-renewal loops.
self.stopping.store(true, Ordering::Release);
let deadline = Instant::now() + self.config.shutdown_timeout;
while !active.is_empty() {
if Instant::now() >= deadline {
warn!("graceful shutdown timeout reached; aborting remaining task attempts");
active.abort_all();
break;
}
tokio::select! {
_ = active.join_next() => {}
_ = tokio::time::sleep(Duration::from_millis(50)) => {}
}
}
Ok(())
}
}
async fn process_lease(
store: Arc<dyn TaskStore>,
handler: Arc<dyn TaskHandler>,
owner: String,
lease: TaskLease,
lease_ms: i64,
renewal_interval: Duration,
stopping: Arc<AtomicBool>,
) {
let task_id = lease.record.task_id;
let fence = lease.fence;
let cancelled = Arc::new(AtomicBool::new(false));
// Pre-dispatch revalidation under the fresh lease.
if let Err(error) = handler.revalidate(&lease).await {
// A transient authority-store outage is not evidence that the work is invalid: leave the
// lease to expire so the bounded recovery scan can retry it without dispatching now.
if matches!(
&error,
TaskWorkerError::Store(TaskError::Unavailable { .. })
) {
debug!(task_id = %task_id, error = %error, "transient revalidation failure; leaving work for recovery");
return;
}
debug!(task_id = %task_id, error = %error, "task revalidation failed; finishing without dispatch");
finish_failed(&store, &lease, "revalidation_failed").await;
return;
}
if stopping.load(Ordering::Acquire) {
// Graceful shutdown before the durable dispatch checkpoint: no outbound call can have
// happened, so the Task is left leased. The bounded recovery scan requeues it after the
// lease expires instead of turning a never-started attempt into a user-visible failure.
return;
}
let attempt_id = format!("attempt-{}", Uuid::now_v7());
// Background lease renewal covers the whole bounded retry budget, including the pinned backoff.
// A durable cancellation clears the lease, so the next renewal fails and the local work stops.
let renewal_store = store.clone();
let renewal_owner = owner.clone();
let renewal_cancelled = cancelled.clone();
let renewal_stopping = stopping.clone();
let renewal_task_id = task_id;
let renewal_handle = tokio::spawn(async move {
let mut interval = tokio::time::interval(renewal_interval);
interval.tick().await;
loop {
interval.tick().await;
if renewal_cancelled.load(Ordering::Acquire) {
break;
}
if renewal_stopping.load(Ordering::Acquire) {
renewal_cancelled.store(true, Ordering::Release);
break;
}
if renewal_store
.renew_lease(&renewal_owner, renewal_task_id, fence, lease_ms)
.await
.is_err()
{
renewal_cancelled.store(true, Ordering::Release);
break;
}
}
});
// The parent span names the leased Task generation; every individual outbound attempt adds its own
// `adapter_attempt` child span with the durable attempt ID committed by its checkpoint.
let span = tracing::info_span!("durable_task_attempt", task_id = %task_id, lease_ref = %attempt_id, fence);
if let Ok(parent) = crank_core::TraceContext::parse(&lease.record.support.traceparent) {
crank_trace::set_parent_from_trace_context(&span, &parent);
}
// Every durable write of this Task (checkpoint, attempt resolution, terminal state) happens inside
// the executor's single retry loop under this lease/fence; the worker only renews and stops.
let context = TaskAttemptContext::new(
store.clone(),
lease.clone(),
cancelled.clone(),
stopping.clone(),
);
let outcome = handler.execute(context).instrument(span).await;
cancelled.store(true, Ordering::Release);
renewal_handle.abort();
match outcome {
TaskAttemptOutcome::Completed => {
debug!(task_id = %task_id, "durable task completed");
}
TaskAttemptOutcome::Failed { code } => {
debug!(task_id = %task_id, code = %code, "durable task failed");
}
TaskAttemptOutcome::OutcomeUnknown => {
warn!(task_id = %task_id, "durable task outcome is uncertain; no automatic retry");
}
TaskAttemptOutcome::WaitingForInput => {
debug!(task_id = %task_id, "durable task suspended until validated input arrives");
}
TaskAttemptOutcome::Stopped => {
debug!(task_id = %task_id, "durable task attempt stopped without a new outcome");
}
}
}
async fn finish_failed(store: &Arc<dyn TaskStore>, lease: &TaskLease, code: &str) {
// The store owns the fate of a refused pre-dispatch attempt: a Task that already carries a proven
// ambiguity still ends `outcome_unknown`, never a plain failure (Story 4.4).
let _ = store.refuse_attempt(lease, code).await;
}
+366 -6
View File
@@ -13,11 +13,14 @@
use std::sync::Arc;
use std::time::Instant;
use crank_core::PlatformApiKeyId;
use crank_core::{
CorrelationContext, TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS,
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
TaskError, TaskId, TaskOwnerKind, TaskPins, TaskRecord, TaskStore, TaskSupportIds,
ToolExecutionMode, current_unix_ms, validate_task_envelope,
CorrelationContext, TASK_DEFAULT_RETENTION_MS, TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES,
TASK_EXECUTABLE_INPUT_MAX_BYTES, TASK_MAX_RETENTION_MS, TASK_SCOPED_KEY_MAX_BYTES,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskId,
TaskInputUpdateOutcome, TaskInputUpdateRequest, TaskLease, TaskOwnerKind, TaskPins, TaskRecord,
TaskStore, TaskSupportIds, ToolExecutionMode, contains_sensitive_task_keys, current_unix_ms,
validate_task_envelope,
};
use sha2::{Digest, Sha256};
@@ -29,6 +32,9 @@ pub struct TaskAcceptDraft {
pub owner_id: String,
/// Server-derived authorization scope identity of the verified principal.
pub authorization_scope: String,
/// Typed immutable Platform API key identity of the verified machine credential. It is sealed
/// into the executable-input binding and revalidated authoritatively by the worker.
pub platform_api_key_id: Option<PlatformApiKeyId>,
pub operation_id: String,
pub operation_version: u32,
pub policy_mode: ToolExecutionMode,
@@ -42,6 +48,8 @@ pub struct TaskAcceptDraft {
/// Retention after the execution deadline.
pub retention_ms: i64,
pub approval_reference: Option<String>,
/// Bounded retry snapshot pinned from the exact published operation at acceptance.
pub retry: crank_core::TaskRetryPolicy,
}
/// Projection of the durable status/result read. `result`/`error` are only present once a terminal
@@ -68,11 +76,22 @@ impl TaskOutcome {
#[derive(Clone)]
pub struct TaskService {
store: Arc<dyn TaskStore>,
crypto: Option<Arc<crate::SecretCrypto>>,
}
impl TaskService {
pub fn new(store: Arc<dyn TaskStore>) -> Self {
Self { store }
Self {
store,
crypto: None,
}
}
pub fn with_secret_crypto(store: Arc<dyn TaskStore>, crypto: Arc<crate::SecretCrypto>) -> Self {
Self {
store,
crypto: Some(crypto),
}
}
/// Readiness of the mandatory durable Task state.
@@ -83,18 +102,61 @@ impl TaskService {
/// Durable acceptance: the returned identity exists only after the store committed it.
pub async fn accept(&self, draft: TaskAcceptDraft) -> Result<TaskAcceptance, TaskError> {
validate_task_envelope(&draft.input)?;
if contains_sensitive_task_keys(&draft.input) {
return Err(TaskError::invalid(
"task executable input contains sensitive fields",
));
}
let (scoped_key, input_fingerprint) = (
scoped_key_digest(&draft)?,
canonical_input_fingerprint(&draft.input)?,
);
let execution_deadline_unix_ms = deadline_unix_ms(draft.deadline);
let retention_ms = draft.retention_ms.clamp(0, TASK_MAX_RETENTION_MS);
let task_id = TaskId::generate();
let crypto = self.crypto.as_ref().ok_or_else(|| {
TaskError::invalid("secret crypto is required for durable Task acceptance")
})?;
let binding = serde_json::json!({
"task_id": task_id.to_string(),
"workspace_id": draft.workspace_id,
"owner_kind": draft.owner_kind.as_str(),
"owner_id": draft.owner_id,
"authorization_scope": draft.authorization_scope,
"platform_api_key_id": draft
.platform_api_key_id
.as_ref()
.map(PlatformApiKeyId::as_str),
"operation_id": draft.operation_id,
"operation_version": draft.operation_version,
"policy_mode": draft.policy_mode,
"input_fingerprint": input_fingerprint,
"input": draft.input,
});
let ciphertext = crypto
.encrypt(&binding)
.map_err(|error| TaskError::invalid(error.to_string()))?;
if ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES {
return Err(TaskError::invalid(
"encrypted task input exceeds its byte budget",
));
}
let (
executable_input_ciphertext,
executable_input_key_version,
executable_input_master_key_epoch,
) = (
Some(ciphertext),
Some(crypto.key_version().to_owned()),
Some(crypto.master_key_epoch()),
);
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
task_id,
workspace_id: draft.workspace_id,
owner_kind: draft.owner_kind,
owner_id: draft.owner_id,
authorization_scope: draft.authorization_scope,
platform_api_key_id: draft.platform_api_key_id,
operation_id: draft.operation_id,
pins: TaskPins {
operation_version: draft.operation_version,
@@ -102,6 +164,7 @@ impl TaskService {
execution_deadline_unix_ms,
retain_until_unix_ms: execution_deadline_unix_ms.saturating_add(retention_ms),
approval_reference: draft.approval_reference,
retry: draft.retry,
},
scoped_key,
input_fingerprint,
@@ -110,6 +173,9 @@ impl TaskService {
draft.correlation.request_id().clone(),
draft.correlation.trace_context(),
),
executable_input_ciphertext,
executable_input_key_version,
executable_input_master_key_epoch,
};
self.store.accept(request).await
}
@@ -134,6 +200,144 @@ impl TaskService {
))
}
/// Owner-scoped cancellation of one accepted Task. The store decides under a row lock whether the
/// Task can still be cancelled or whether an already checkpointed attempt makes the outcome
/// uncertain; the service only carries the request and a bounded reason.
pub async fn cancel(
&self,
access: &TaskAccess,
task_id: TaskId,
expected_version: Option<u32>,
reason: &str,
) -> Result<crank_core::TaskCancelOutcome, TaskError> {
let request = crank_core::TaskCancelRequest {
access: access.clone(),
task_id,
expected_version,
reason: reason.to_owned(),
};
request.validate()?;
self.store.cancel(request).await
}
/// Bounded owner-scoped attempt history of one Task.
pub async fn attempts(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<Vec<crank_core::TaskAttempt>, TaskError> {
self.store.attempts(access, task_id).await
}
/// Decrypts and validates the executable input bound to a valid current lease.
pub async fn executable_input(
&self,
lease: &TaskLease,
) -> Result<serde_json::Value, TaskError> {
let crypto = self.crypto.as_ref().ok_or_else(|| {
TaskError::invalid("secret crypto is required for durable Task execution")
})?;
let (ciphertext, key_version, epoch) = self
.store
.read_executable_input(&lease.owner, lease.record.task_id, lease.fence)
.await?;
if ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES {
return Err(TaskError::invalid(
"task executable input ciphertext is invalid",
));
}
let binding = crypto
.decrypt_for_epoch(&key_version, epoch, &ciphertext)
.map_err(|error| TaskError::invalid(error.to_string()))?;
let object = binding
.as_object()
.ok_or_else(|| TaskError::invalid("task executable input binding is invalid"))?;
let check_string = |field: &str, expected: &str| -> Result<(), TaskError> {
match object.get(field).and_then(|value| value.as_str()) {
Some(actual) if actual == expected => Ok(()),
_ => Err(TaskError::invalid("task executable input binding mismatch")),
}
};
check_string("task_id", &lease.record.task_id.to_string())?;
check_string("workspace_id", &lease.record.workspace_id)?;
check_string("owner_id", &lease.record.owner_id)?;
check_string("authorization_scope", &lease.record.authorization_scope)?;
let expected_key = lease
.record
.platform_api_key_id
.as_ref()
.map(PlatformApiKeyId::as_str);
match (object.get("platform_api_key_id"), expected_key) {
(None | Some(serde_json::Value::Null), None) => {}
(Some(serde_json::Value::String(actual)), Some(expected)) if actual == expected => {}
_ => {
return Err(TaskError::invalid("task executable input binding mismatch"));
}
}
check_string("operation_id", &lease.record.operation_id)?;
match object.get("owner_kind").and_then(|value| value.as_str()) {
Some(actual) if actual == lease.record.owner_kind.as_str() => {}
_ => return Err(TaskError::invalid("task executable input binding mismatch")),
}
match object
.get("operation_version")
.and_then(|value| value.as_u64())
{
Some(actual) if actual == u64::from(lease.record.pins.operation_version) => {}
_ => return Err(TaskError::invalid("task executable input binding mismatch")),
}
let expected_policy = match lease.record.pins.policy_mode {
ToolExecutionMode::Synchronous => "synchronous",
ToolExecutionMode::Background => "background",
ToolExecutionMode::Adaptive => "adaptive",
};
match object.get("policy_mode").and_then(|value| value.as_str()) {
Some(actual) if actual == expected_policy => {}
_ => return Err(TaskError::invalid("task executable input binding mismatch")),
}
check_string("input_fingerprint", &lease.record.input_fingerprint)?;
let base_input = object
.get("input")
.cloned()
.ok_or_else(|| TaskError::invalid("task executable input binding is missing input"))?;
if canonical_input_fingerprint(&base_input)? != lease.record.input_fingerprint {
return Err(TaskError::invalid("task executable input binding mismatch"));
}
// A committed continuation replaces only the effective payload. The acceptance fingerprint and
// the original input stay in the binding, so an original accept replay is still a replay.
let effective_input = match object.get("continuation") {
Some(continuation) => {
let continuation = continuation
.as_object()
.ok_or_else(|| TaskError::invalid("task input continuation is invalid"))?;
let effective = continuation
.get("effective_input_fingerprint")
.and_then(|value| value.as_str())
.ok_or_else(|| TaskError::invalid("task input continuation is invalid"))?;
if effective != lease.record.effective_input_fingerprint {
return Err(TaskError::invalid(
"task input continuation fingerprint mismatch",
));
}
let input = continuation.get("input").cloned().ok_or_else(|| {
TaskError::invalid("task input continuation is missing input")
})?;
if canonical_input_fingerprint(&input)? != effective {
return Err(TaskError::invalid("task input continuation is invalid"));
}
input
}
None => {
if lease.record.effective_input_fingerprint != lease.record.input_fingerprint {
return Err(TaskError::invalid("task input continuation is missing"));
}
base_input
}
};
validate_executable_payload(&effective_input)?;
Ok(effective_input)
}
/// Default retention pinned by the application layer when it does not choose one explicitly.
pub fn default_retention_ms() -> i64 {
TASK_DEFAULT_RETENTION_MS
@@ -159,6 +363,105 @@ impl TaskService {
pub fn input_fingerprint(input: &serde_json::Value) -> Result<String, TaskError> {
canonical_input_fingerprint(input)
}
/// Applies one additional validated input to a suspended Task (Story 4.5).
///
/// The stored native schema snapshot is the only validation authority; the canonical digest binds
/// the effective payload, and the existing encrypted executable-input binding is extended with a
/// continuation segment instead of a second cipher store. The store transaction owns authority,
/// receipt, generation, TTL and CAS, so an identical replay returns the committed outcome.
pub async fn update_input(
&self,
access: &TaskAccess,
task_id: TaskId,
wait_id: &str,
expected_version: u32,
update_key: &str,
input: &serde_json::Value,
) -> Result<TaskInputUpdateOutcome, TaskError> {
access.validate()?;
validate_task_envelope(input)?;
if contains_sensitive_task_keys(input) {
return Err(TaskError::invalid(
"task executable input contains sensitive fields",
));
}
let canonical_fingerprint = canonical_input_fingerprint(input)?;
let pending = self.store.input_request(access, task_id).await?;
if pending.wait_id != wait_id {
return Err(TaskError::StateConflict);
}
// The immutable snapshot, never the current publication, is the schema authority.
let schema: crank_schema::Schema = serde_json::from_value(pending.schema_snapshot.clone())
.map_err(|_| TaskError::unavailable("stored task input schema is invalid"))?;
schema
.validate_shape(input)
.map_err(|_| TaskError::invalid("task input does not match the pinned input schema"))?;
let update_key_digest = input_update_key_digest(access, task_id, wait_id, update_key)?;
let crypto = self.crypto.as_ref().ok_or_else(|| {
TaskError::invalid("secret crypto is required for durable Task continuation")
})?;
let (ciphertext, key_version, epoch) = self
.store
.read_executable_input_for_update(access, task_id)
.await?;
let binding = crypto
.decrypt_for_epoch(&key_version, epoch, &ciphertext)
.map_err(|error| TaskError::invalid(error.to_string()))?;
let mut object = binding
.as_object()
.cloned()
.ok_or_else(|| TaskError::invalid("task executable input binding is invalid"))?;
check_binding_string(&object, "task_id", &task_id.to_string())?;
check_binding_string(&object, "workspace_id", &access.workspace_id)?;
check_binding_string(&object, "owner_id", &access.owner_id)?;
check_binding_string(&object, "authorization_scope", &access.authorization_scope)?;
let base_input = object
.get("input")
.cloned()
.ok_or_else(|| TaskError::invalid("task executable input binding is missing input"))?;
let base_fingerprint = object
.get("input_fingerprint")
.and_then(|value| value.as_str())
.ok_or_else(|| TaskError::invalid("task executable input binding is invalid"))?;
if canonical_input_fingerprint(&base_input)? != base_fingerprint {
return Err(TaskError::invalid("task executable input binding mismatch"));
}
object.insert(
"binding_format_version".to_owned(),
serde_json::json!(2_u32),
);
object.insert(
"continuation".to_owned(),
serde_json::json!({
"wait_id": wait_id,
"schema_revision": pending.schema_revision,
"effective_input_fingerprint": canonical_fingerprint,
"input": input,
}),
);
let continuation = crypto
.encrypt(&serde_json::Value::Object(object))
.map_err(|error| TaskError::invalid(error.to_string()))?;
if continuation.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES {
return Err(TaskError::invalid(
"encrypted task input exceeds its byte budget",
));
}
let request = TaskInputUpdateRequest {
access: access.clone(),
task_id,
wait_id: wait_id.to_owned(),
expected_version,
update_key_digest,
effective_input_fingerprint: canonical_fingerprint.clone(),
canonical_fingerprint,
executable_input_ciphertext: continuation,
executable_input_key_version: crypto.key_version().to_owned(),
executable_input_master_key_epoch: crypto.master_key_epoch(),
};
self.store.apply_input(request).await
}
}
fn deadline_unix_ms(deadline: Instant) -> i64 {
@@ -209,6 +512,61 @@ fn hex_digest(bytes: &[u8]) -> String {
.collect()
}
fn check_binding_string(
object: &serde_json::Map<String, serde_json::Value>,
field: &str,
expected: &str,
) -> Result<(), TaskError> {
match object.get(field).and_then(|value| value.as_str()) {
Some(actual) if actual == expected => Ok(()),
_ => Err(TaskError::invalid("task executable input binding mismatch")),
}
}
fn validate_executable_payload(input: &serde_json::Value) -> Result<(), TaskError> {
validate_task_envelope(input)?;
let encoded = serde_json::to_vec(input)
.map_err(|_| TaskError::invalid("task executable input is not serializable"))?;
if encoded.len() > TASK_EXECUTABLE_INPUT_MAX_BYTES {
return Err(TaskError::invalid(
"task executable input exceeds its byte budget",
));
}
if contains_sensitive_task_keys(input) {
return Err(TaskError::invalid(
"task executable input contains sensitive fields",
));
}
Ok(())
}
/// Digest of the scoped update key. The key itself never leaves the caller, and the scope covers the
/// exact Task generation so one key cannot be replayed across Tasks or waits.
fn input_update_key_digest(
access: &TaskAccess,
task_id: TaskId,
wait_id: &str,
key: &str,
) -> Result<String, TaskError> {
if key.is_empty()
|| key.len() > TASK_SCOPED_KEY_MAX_BYTES
|| key.bytes().any(|byte| byte.is_ascii_control())
{
return Err(TaskError::invalid("task input update key is invalid"));
}
let material = format!(
"{}\n{}\n{}\n{}\n{}\n{}\n{}",
access.workspace_id,
access.owner_kind.as_str(),
access.owner_id,
access.authorization_scope,
task_id,
wait_id,
key
);
Ok(hex_digest(material.as_bytes()))
}
#[cfg(test)]
mod tests {
use super::*;
@@ -220,6 +578,7 @@ mod tests {
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
platform_api_key_id: Some(PlatformApiKeyId::new("pk_1")),
operation_id: "op_report".to_owned(),
operation_version: 2,
policy_mode: ToolExecutionMode::Background,
@@ -229,6 +588,7 @@ mod tests {
deadline: Instant::now() + std::time::Duration::from_secs(30),
retention_ms: TASK_DEFAULT_RETENTION_MS,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
}
}
+86 -5
View File
@@ -106,6 +106,83 @@ async fn deadline_after_read_only_dispatch_remains_safe_to_retry() {
assert_eq!(failure.stage(), ExecutionStage::Upstream);
}
#[tokio::test]
async fn adapter_retry_child_spans_are_distinct_without_a_durable_attempt() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
let capture = TraceCapture::default();
let subscriber = tracing_subscriber::registry().with(capture.clone());
let call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let executor = RuntimeExecutorBuilder::new()
.register_adapter(Arc::new(FlakyReadAdapter {
call_count: Arc::clone(&call_count),
}))
.build();
let mut operation: crank_runtime::RuntimeOperation = operation().into();
let Target::Rest(target) = &mut operation.target;
target.method = HttpMethod::Get;
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
// The synchronous tools/call path has no durable Attempt: every adapter attempt gets its own
// synthetic id. A Task dispatch instead reuses the durable id (mcp-server `attempt_trace`).
let result = async {
let request_root = tracing::info_span!("request_root");
executor
.execute(&operation, &json!({"name": "retry-read"}))
.instrument(request_root)
.await
}
.with_subscriber(subscriber)
.await
.expect("read retry should recover");
assert_eq!(result, json!({"accepted": true}));
let spans = capture.snapshot();
let runtime_span = spans
.iter()
.find(|span| span.name == "runtime.execute")
.expect("runtime.execute span");
assert_eq!(
runtime_span.parent_name,
Some("request_root"),
"the execution continues the caller's trace root"
);
let attempt_spans: Vec<_> = spans
.iter()
.filter(|span| span.name == "adapter_attempt")
.collect();
assert_eq!(attempt_spans.len(), 2);
for span in &attempt_spans {
assert_eq!(span.parent_name, Some("runtime.execute"));
assert!(
!span
.fields
.get("attempt_id")
.unwrap_or(&"".to_string())
.is_empty()
);
assert!(
!span
.fields
.get("attempt_id")
.unwrap_or(&"".to_string())
.contains("traceparent")
);
}
let attempts: Vec<String> = attempt_spans
.iter()
.map(|span| span.fields.get("attempt").cloned().unwrap_or_default())
.collect();
assert!(attempts.contains(&"1".to_string()));
assert!(attempts.contains(&"2".to_string()));
let ids: Vec<String> = attempt_spans
.iter()
.map(|span| span.fields.get("attempt_id").cloned().unwrap_or_default())
.collect();
assert_ne!(ids[0], ids[1]);
}
#[tokio::test]
async fn read_retry_revalidates_adapter_attempt_and_keeps_one_success_outcome() {
let call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
@@ -236,11 +313,7 @@ async fn successful_execution_has_real_stages_and_omits_inapplicable_ones() {
.find(|span| span.name == "runtime.execute")
.expect("runtime span");
assert_eq!(runtime.parent_name, Some("mcp.request"));
for child in [
"runtime.arguments.map",
"upstream.http",
"runtime.response.transform",
] {
for child in ["runtime.arguments.map", "runtime.response.transform"] {
assert_eq!(
spans
.iter()
@@ -250,6 +323,14 @@ async fn successful_execution_has_real_stages_and_omits_inapplicable_ones() {
"{child} must be a runtime child"
);
}
assert_eq!(
spans
.iter()
.find(|span| span.name == "upstream.http")
.and_then(|span| span.parent_name),
Some("adapter_attempt"),
"upstream.http must be an adapter child"
);
}
#[tokio::test]
+17 -4
View File
@@ -19,11 +19,20 @@ struct TestPostgres {
}
pub async fn postgres_database_url() -> &'static str {
&test_postgres().await.database_url
&test_postgres("local-tests").await.database_url
}
pub async fn postgres_schema_url(prefix: &str) -> String {
let database_url = postgres_database_url().await;
create_schema_url(prefix, database_url).await
}
/// The first caller owns the process-wide disposable PostgreSQL container.
pub async fn postgres_schema_url_for_task(prefix: &str, task: &str) -> String {
create_schema_url(prefix, &test_postgres(task).await.database_url).await
}
async fn create_schema_url(prefix: &str, database_url: &str) -> String {
let schema = unique_schema_name(prefix);
let mut connection = connect_admin_with_retry(database_url).await;
@@ -73,22 +82,26 @@ pub fn unique_schema_name(prefix: &str) -> String {
format!("{}_{}", normalized_prefix, uuid::Uuid::now_v7().simple())
}
async fn test_postgres() -> &'static TestPostgres {
async fn test_postgres(task: &str) -> &'static TestPostgres {
POSTGRES
.get_or_init(|| async {
tokio::task::spawn_blocking(start_postgres)
let task = task.to_owned();
tokio::task::spawn_blocking(move || start_postgres(task))
.await
.expect("test PostgreSQL container startup task must complete")
})
.await
}
fn start_postgres() -> TestPostgres {
fn start_postgres(task: String) -> TestPostgres {
let container = Postgres::default()
.with_db_name("crank")
.with_user("crank")
.with_password("crank")
.with_tag("16-alpine")
.with_label("hermes.disposable", "true")
.with_label("hermes.task", task)
.with_label("hermes.board", "crank")
.start()
.expect("test PostgreSQL container must start");
register_container_cleanup(container.id().to_owned());
Loaded 100 of 112 files, more files were not shown because too many files have changed in this diff. Show more