Files
crank/tests/unit/test_project_advertised_capabilities.py
bsodfather 54ce770682
CI / Rust Checks (pull_request) Failing after 13m41s
CI / UI Checks (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Deploy (pull_request) Skipped
Community Release / release (push) Failing after 13m33s
feat(epic3): deliver the accepted Epic 3 scope through Story 3.17
Recovery delivery of the accumulated, TL-accepted Epic 3 work that had never been
committed to Git (stories 3.6-3.17, see the crank board). Story 3.18 is not accepted
yet and is deliberately excluded: its logging surface, its capability registration,
its oracle projections and its tests are not part of this tree.

Accepted scope
- 3.6 typed static Resource Drafts; 3.7 publication and exact binding of static
  Resource Versions; 3.8 MCP discovery/read and revocation of static Resources;
- 3.9 Resource import/export in API and UI plus TLS dependency maintenance;
  3.10 REST-backed Resource templates; 3.11 shared REST read runtime and the MCP
  revision/capability foundation;
- 3.12 parameterised Prompt Drafts with preview; 3.13 Prompt publication and exact
  version binding to Agents; 3.14 MCP prompts; 3.15 Prompt import/export;
- 3.16 composite Prompts with published Resources (dependency content scan plus
  dynamic read through the accepted 3.11 runtime, admin preview and MCP prompts/get);
- 3.17 bounded MCP completion suggestions from typed published schemas and the
  resources/list admission fix for bound template publications.

Accepted evidence being reused (unchanged code paths)
- 3.16 candidate: core 7, admin-api 22, registry 4, mcp-server 4, UI 6; one final
  `just verify` run 1651 passed / 0 failed / 2 ignored.
- 3.17 candidate: full run 1672 passed / 0 failed / 2 ignored, QA targeted reruns and
  a real MCP Inspector 2.12.0 client smoke.

Delivery engineering in this commit
- CI: one event per change (feature branches are gated by their pull request, push
  runs only on main), story events run the changed packages, the full workspace suite
  stays the feature/release gate; no cache/sharding without a measured runner check.
- Release: the packaged archives, checksums and SBOM are now uploaded to the Gitea
  release entry of the tag instead of dying with the runner workspace.

Not included on purpose: Story 3.18 work, build caches, runtime databases, local
credentials and scratch evidence.

Refs: crank kanban t_e39d7263; baseline 1b6db09bd1.
2026-10-25 09:08:22 +00:00

345 lines
16 KiB
Python

"""Story 3.1 ATDD (red phase): the per-era advertised-capabilities projection artifact (AT-13).
Oracle contract declared by this stage (the implementation stage owns the internals):
scripts/project-advertised-capabilities.py
--inventory <path> --revision <2026-07-28|2025-11-25>
--frozen <path> --provenance <path>
The script reads ONLY the canonical inventory (`protocol` block), applies the server-only projection
rules 1-4 of the accepted spec section 5, and writes
* `--frozen` the canonical capabilities payload: one JSON object, UTF-8, compact (`:`/`,`),
object keys sorted byte-lexicographically, no trailing newline;
* `--provenance` a sidecar with `inventory_sha256`, the projection `rule` id and the participating
identity triples. The sidecar is audit-only and is never byte-compared with the
endpoint.
It must never call the MCP server, must never import the registry and must never read a captured
server response: the expectation is derived from the reviewed inventory alone.
Why these tests are guarded: the executable does not exist on this revision, so a missing executable
is an honest ARTIFACT gap (`skip`), NOT a behavioural red and NOT a pass. The moment the script
lands, the same assertions run unchanged - that is the point of writing them now.
"""
from __future__ import annotations
import copy
import hashlib
import json
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "project-advertised-capabilities.py"
FIXTURES = ROOT / "tests" / "fixtures" / "mcp-revisions"
CANDIDATE = FIXTURES / "inventory-reviewed-v2-candidate.json"
MODERN = "2026-07-28"
LEGACY = "2025-11-25"
# The reviewed v2 candidate is a frozen Story 3.1 input: its Prompt capability is still deferred, so
# its own projection stays `resources`+`tools`.
EXPECTED_FROZEN = (
b'{"resources":{"listChanged":false,"subscribe":false},"tools":{"listChanged":false}}'
)
EXPECTED_SHA256 = "e38c795fc28fee74168cb045427df55d882f8f7ace206af5c9a3ca4df5892c9a"
# The canonical inventory implements the completions/prompts/resources/tools surfaces, so the committed
# oracle and the canonical projection carry exactly those four; the logging surface is still deferred
# to its owning story and must not appear in the advertisement.
COMMITTED_FROZEN = (
b'{"completions":{},"prompts":{"listChanged":false},"resources":{"listChanged":false,"subscribe":false},"tools":{"listChanged":false}}'
)
COMMITTED_SHA256 = "08c29e3df2bd8ffc58c1ec172714cc06031c5b279ef042940a8591940aca91fe"
class AdvertisedCapabilitiesProjectionTests(unittest.TestCase):
def setUp(self) -> None:
if not SCRIPT.is_file():
self.skipTest(
"artifact gap, not a behavioural red: scripts/project-advertised-capabilities.py is "
"created by the implementation stage (BUILD); the assertions below are ready and run "
"unchanged once the executable exists"
)
self._temporary_directory = tempfile.TemporaryDirectory()
self.root = Path(self._temporary_directory.name)
self.addCleanup(self._temporary_directory.cleanup)
def load_candidate(self) -> dict:
return json.loads(CANDIDATE.read_text(encoding="utf-8"))
def project(self, inventory: dict, revision: str, stem: str) -> subprocess.CompletedProcess[str]:
inventory_path = self.root / f"{stem}.inventory.json"
inventory_path.write_text(json.dumps(inventory), encoding="utf-8")
return subprocess.run(
[
sys.executable,
str(SCRIPT),
"--inventory",
str(inventory_path),
"--revision",
revision,
"--frozen",
str(self.root / f"{stem}.frozen.json"),
"--provenance",
str(self.root / f"{stem}.provenance.json"),
],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=str(ROOT),
)
def frozen(self, stem: str) -> Path:
return self.root / f"{stem}.frozen.json"
def test_positive_projection_equals_the_committed_frozen_oracle_for_both_eras(self) -> None:
candidate = self.load_candidate()
for revision in (MODERN, LEGACY):
with self.subTest(revision=revision):
result = self.project(candidate, revision, f"positive-{revision}")
combined = result.stdout + result.stderr
self.assertEqual(result.returncode, 0, combined)
produced = self.frozen(f"positive-{revision}").read_bytes()
self.assertEqual(produced, EXPECTED_FROZEN)
# The committed oracle is projected from the canonical inventory, which now also
# implements the Prompt vertical (Story 3.14); the reviewed v2 candidate is a frozen
# Story 3.1 input and is intentionally no longer byte-equal to it. The canonical
# inventory -> committed oracle equality is pinned by
# `test_r10_canonical_inventory_projects_to_the_committed_oracle_and_binds_its_digest`.
provenance = json.loads(
(self.root / f"positive-{revision}.provenance.json").read_text(encoding="utf-8")
)
self.assertTrue(provenance["rule"])
self.assertTrue(provenance["inventory_sha256"])
self.assertEqual(
sorted(
(item["role"], item["kind"], item["surface"])
for item in provenance["projection_inputs"]
),
[("server", "capability", "resources"), ("server", "capability", "tools")],
)
def test_client_role_capability_is_never_projected(self) -> None:
candidate = self.load_candidate()
for entry in candidate["protocol"]["capabilities"]:
if entry["surface"] == "elicitation" and entry["role"] == "client":
entry["status"] = "supported"
entry["advertised"] = {"elicitation": {"form": {}}}
for revision in (MODERN, LEGACY):
with self.subTest(revision=revision):
result = self.project(candidate, revision, f"client-role-{revision}")
combined = result.stdout + result.stderr
self.assertEqual(result.returncode, 0, combined)
self.assertEqual(self.frozen(f"client-role-{revision}").read_bytes(), EXPECTED_FROZEN)
def test_per_revision_override_changes_only_that_revision(self) -> None:
candidate = self.load_candidate()
for entry in candidate["protocol"]["capabilities"]:
if entry["surface"] == "tools" and entry["role"] == "server":
entry.setdefault("per_revision", {})[MODERN] = {"status": "deferred",
"reason": "modern projection is empty"}
modern = self.project(candidate, MODERN, "override-modern")
self.assertEqual(modern.returncode, 0, modern.stdout + modern.stderr)
self.assertEqual(
self.frozen("override-modern").read_bytes(),
b'{"resources":{"listChanged":false,"subscribe":false}}',
"deferring tools for the modern era leaves the modern resources capability in place",
)
legacy = self.project(candidate, LEGACY, "override-legacy")
self.assertEqual(legacy.returncode, 0, legacy.stdout + legacy.stderr)
self.assertEqual(self.frozen("override-legacy").read_bytes(), EXPECTED_FROZEN)
def test_conflicting_top_level_advertised_keys_are_rejected(self) -> None:
candidate = self.load_candidate()
conflicting = copy.deepcopy(candidate["protocol"]["capabilities"][0])
conflicting.update(
{
"surface": "logging",
"kind": "capability",
"role": "server",
"status": "supported",
"advertised": {"tools": {"listChanged": True}},
}
)
candidate["protocol"]["capabilities"].append(conflicting)
result = self.project(candidate, MODERN, "conflict")
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertFalse(self.frozen("conflict").exists())
def test_advertised_leaf_not_declared_by_subfeatures_is_rejected(self) -> None:
candidate = self.load_candidate()
for entry in candidate["protocol"]["capabilities"]:
if entry["surface"] == "tools" and entry["role"] == "server":
entry["advertised"] = {"tools": {"listChanged": False, "unexpected": True}}
result = self.project(candidate, MODERN, "undeclared-subfeature")
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertFalse(self.frozen("undeclared-subfeature").exists())
def test_supported_server_capability_without_advertised_is_rejected(self) -> None:
candidate = self.load_candidate()
for entry in candidate["protocol"]["capabilities"]:
if entry["surface"] == "tools" and entry["role"] == "server":
entry["advertised"] = None
result = self.project(candidate, MODERN, "missing-advertised")
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertFalse(self.frozen("missing-advertised").exists())
# --- Story 3.1 #8 remaining-foundation regressions (R3 producer side, R10 canonical chain) -----
# The validator (Gate A) and the producer must reject the same malformed advertisement shape
# independently: a scalar fragment and a top-level key that does not belong to the declaring
# surface both produce a `capabilities` object that is not the reviewed projection.
def test_r3_scalar_advertised_fragment_is_rejected_by_the_producer(self) -> None:
candidate = self.load_candidate()
for entry in candidate["protocol"]["capabilities"]:
if entry["surface"] == "tools" and entry["role"] == "server":
entry["advertised"] = {"tools": True}
result = self.project(candidate, MODERN, "r3-scalar-fragment")
self.assertNotEqual(
result.returncode,
0,
"a scalar advertisement fragment cannot be a canonical capabilities object: "
+ result.stdout
+ result.stderr,
)
self.assertFalse(self.frozen("r3-scalar-fragment").exists())
def test_r3_foreign_top_level_advertised_key_is_rejected_by_the_producer(self) -> None:
candidate = self.load_candidate()
for entry in candidate["protocol"]["capabilities"]:
if entry["surface"] == "tools" and entry["role"] == "server":
entry["advertised"] = {"prompts": {"listChanged": False}}
result = self.project(candidate, MODERN, "r3-foreign-key")
self.assertNotEqual(
result.returncode,
0,
"an advertised top-level key that does not belong to the declaring surface must be "
"rejected: " + result.stdout + result.stderr,
)
self.assertFalse(self.frozen("r3-foreign-key").exists())
def test_r10_canonical_inventory_projects_to_the_committed_oracle_and_binds_its_digest(self) -> None:
"""The reproducible chain: canonical inventory -> projection -> committed frozen bytes.
The producer is run on the real canonical artifact (`docs/capability-inventory.json`), so the
frozen bytes and the sidecar digest are derived from the canonical source rather than from a
fixture. The committed sidecar itself is pinned by
`test_story31_foundation_boundaries.py` (it must record the canonical digest).
"""
canonical = ROOT / "docs" / "capability-inventory.json"
self.assertTrue(canonical.is_file(), "the canonical inventory must exist")
canonical_digest = hashlib.sha256(canonical.read_bytes()).hexdigest()
for revision in (MODERN, LEGACY):
with self.subTest(revision=revision):
inventory_path = self.root / f"canonical-{revision}.inventory.json"
inventory_path.write_bytes(canonical.read_bytes())
result = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--inventory",
str(inventory_path),
"--revision",
revision,
"--frozen",
str(self.root / f"canonical-{revision}.frozen.json"),
"--provenance",
str(self.root / f"canonical-{revision}.provenance.json"),
],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=str(ROOT),
)
combined = result.stdout + result.stderr
self.assertEqual(result.returncode, 0, combined)
produced = (self.root / f"canonical-{revision}.frozen.json").read_bytes()
committed = (FIXTURES / f"{revision}-advertised-capabilities.frozen.json").read_bytes()
self.assertEqual(
produced,
committed,
"the canonical inventory must project to the committed frozen oracle",
)
provenance = json.loads(
(self.root / f"canonical-{revision}.provenance.json").read_text(encoding="utf-8")
)
self.assertEqual(
provenance["inventory_sha256"],
canonical_digest,
"the producer must record the digest of the inventory it actually read",
)
self.assertEqual(
sorted(
(item["role"], item["kind"], item["surface"])
for item in provenance["projection_inputs"]
),
[
("server", "capability", "completions"),
("server", "capability", "prompts"),
("server", "capability", "resources"),
("server", "capability", "tools"),
],
)
class CommittedFrozenOracleTests(unittest.TestCase):
"""The committed per-era oracle artifacts themselves (runs without the future executable)."""
def test_committed_frozen_oracle_is_canonical_and_matches_its_sidecar(self) -> None:
for revision in (MODERN, LEGACY):
with self.subTest(revision=revision):
frozen = FIXTURES / f"{revision}-advertised-capabilities.frozen.json"
payload = frozen.read_bytes()
self.assertEqual(
payload, COMMITTED_FROZEN
)
self.assertEqual(hashlib.sha256(payload).hexdigest(), COMMITTED_SHA256)
self.assertEqual(
json.dumps(json.loads(payload), sort_keys=True, separators=(",", ":")).encode("utf-8"),
payload,
"the frozen payload must already be in canonical form",
)
sidecar = json.loads(
(FIXTURES / f"{revision}-advertised-capabilities.provenance.json").read_text(
encoding="utf-8"
)
)
self.assertTrue(sidecar["inventory_sha256"])
self.assertTrue(sidecar["rule"])
self.assertEqual(
sorted(
(item["role"], item["kind"], item["surface"])
for item in sidecar["projection_inputs"]
),
[
("server", "capability", "completions"),
("server", "capability", "prompts"),
("server", "capability", "resources"),
("server", "capability", "tools"),
],
"the sidecar must name the exact full projection input set, in any order",
)
if __name__ == "__main__":
unittest.main()