Клиент, согласовавший tools list-changed, получает bounded `notifications/tools/list_changed` после коммита видимой Tool-проекции опубликованного Agent snapshot; `tools/list` остаётся авторитетным для клиента без notification и после reconnect. - signals.rs: третий тип сигнала в существующем stream-local watcher (один tick = максимум один кадр на тип, fixed-capacity queue, без cross-process bus и без post-commit hook); fingerprint сравнивает tool_name/operation.version/title/description, поэтому Tool-only commit не даёт Resource/Prompt сигналов и наоборот. - registry.rs: tools.listChanged=true (реально отправляется в обеих revision); modern filter toolsListChanged теперь подтверждается acknowledge и не шлётся вне согласованного набора. - legacy GET-стрим несёт Tools-сигнал вместе с Resource/Prompt. - ATDD: RED на baseline (ack пуст, Tools-кадра нет, capability false), GREEN — 8 integration-сценариев × обе revision, unit-оракулы wire, representatives official MCP SDK 1.32.2 smoke со штатными headers. - capability inventory/baseline manifest и frozen projection обновлены под tools.listChanged=true; ci-story-checks.sh дополнен AC 3.5.
1325 lines
71 KiB
Python
1325 lines
71 KiB
Python
"""Story 3.1 #8 — remaining-foundation regression package (test-only, no production fix).
|
|
|
|
This stage is *only* the executable correction of the TL remaining-foundation review
|
|
(`build/remaining-ac-review/TL-REVIEW.md`, groups R1-R10) after the BUILD card. Every test here
|
|
pins the **agreed target contract** of the accepted SPEC (`build/remaining-ac-spec/remaining-ac-spec.md`
|
|
sections 5, 6.1-6.2, 7.1-7.2, 8) and is expected to be **RED on the current revision**: the validator
|
|
accepts structurally invalid documents, the projection is not era-indexed, the registration does not
|
|
drive the real dispatch and the shared Tools call boundary / semantic observer do not exist yet.
|
|
|
|
Honesty rules of this stage:
|
|
|
|
* a behavioural RED is observed on the real executable (`scripts/validate-capability-inventory.py`)
|
|
as a subprocess, with a one-factor mutation of the reviewed candidate as the only input change;
|
|
* the unmutated reviewed candidate is the independent **positive control** for every negative here
|
|
(`test_control_reviewed_candidate_keeps_the_truthful_gate_verdicts`);
|
|
* a seam that does not exist on this revision (registry dispatch adoption, era-indexed
|
|
advertisement, the neutral tools/call boundary, the semantic era/service markers) is recorded as an
|
|
explicit **GAP** with the exact future test + oracle, and - where a source-level contract is
|
|
reasonable - as an executable source check, never as a fabricated RED, a `skip`-as-pass, or a
|
|
compile failure of the accepted build.
|
|
|
|
Oracle inputs are the reviewed v2 candidate fixture and its bounded review mutations; expectations are
|
|
read off the reviewed tables (`surface-inventory.md` sections 3-7), not copied from current output.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
VALIDATOR = ROOT / "scripts" / "validate-capability-inventory.py"
|
|
PRODUCER = ROOT / "scripts" / "project-advertised-capabilities.py"
|
|
CANONICAL_SCHEMA = ROOT / "docs" / "schemas" / "capability-inventory.schema.json"
|
|
CANONICAL_INVENTORY = ROOT / "docs" / "capability-inventory.json"
|
|
MCP_REVISION_FIXTURES = ROOT / "tests" / "fixtures" / "mcp-revisions"
|
|
CANDIDATE = MCP_REVISION_FIXTURES / "inventory-reviewed-v2-candidate.json"
|
|
MCP_CRATE_SRC = ROOT / "crates" / "crank-community-mcp" / "src"
|
|
REGISTRY_RS = MCP_CRATE_SRC / "protocol" / "registry.rs"
|
|
COMMAND_RS = MCP_CRATE_SRC / "protocol" / "command.rs"
|
|
|
|
MODERN = "2026-07-28"
|
|
LEGACY = "2025-11-25"
|
|
|
|
# The reviewed applicable-identity basis (independent of the fixture, read off surface-inventory.md
|
|
# sections 3-7). Used only by the per-era completeness expectations.
|
|
REVIEWED_TOTAL_IDENTITIES = 50
|
|
REVIEWED_MODERN_APPLICABLE = 35
|
|
REVIEWED_LEGACY_APPLICABLE = 43
|
|
|
|
# Diagnostic substrings that are part of the agreed target behaviour, not of the current output.
|
|
REJECTED_BOTH_MODES = "invalid in BOTH modes before Gate B"
|
|
|
|
|
|
class Story31FoundationBoundaryTests(unittest.TestCase):
|
|
"""R1-R4, R10: executable regressions on the real validator/producer."""
|
|
|
|
def setUp(self) -> None:
|
|
if not VALIDATOR.is_file():
|
|
self.fail(f"the validator under regression must exist: {VALIDATOR}")
|
|
self._temporary_directory = tempfile.TemporaryDirectory()
|
|
self.root = Path(self._temporary_directory.name)
|
|
self.addCleanup(self._temporary_directory.cleanup)
|
|
(self.root / "docs" / "schemas").mkdir(parents=True, exist_ok=True)
|
|
shutil.copyfile(CANONICAL_SCHEMA, self.root / "docs" / "schemas" / "capability-inventory.schema.json")
|
|
|
|
# --- fixtures / helpers --------------------------------------------------------------------
|
|
|
|
def candidate(self) -> dict[str, Any]:
|
|
return json.loads(CANDIDATE.read_text(encoding="utf-8"))
|
|
|
|
@staticmethod
|
|
def tools_entry(payload: dict[str, Any]) -> dict[str, Any]:
|
|
return next(
|
|
cell
|
|
for cell in payload["protocol"]["capabilities"]
|
|
if cell["surface"] == "tools" and cell["role"] == "server"
|
|
)
|
|
|
|
@staticmethod
|
|
def revision_entry(payload: dict[str, Any], revision: str) -> dict[str, Any]:
|
|
return next(entry for entry in payload["protocol"]["revisions"] if entry["revision"] == revision)
|
|
|
|
def run_validator(
|
|
self,
|
|
payload: dict[str, Any],
|
|
mode: str,
|
|
report: bool = False,
|
|
) -> tuple[subprocess.CompletedProcess[str], Path]:
|
|
inventory = self.root / "docs" / "capability-inventory.json"
|
|
inventory.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
for flow in payload.get("flows", []):
|
|
for kind in ("automated", "manual"):
|
|
for relative in flow.get("evidence", {}).get(kind, []):
|
|
evidence = self.root / relative
|
|
evidence.parent.mkdir(parents=True, exist_ok=True)
|
|
if not evidence.exists():
|
|
evidence.write_text("reviewed evidence\n", encoding="utf-8")
|
|
report_path = self.root / f"report-{mode}.json"
|
|
command = [
|
|
sys.executable,
|
|
str(VALIDATOR),
|
|
"--root",
|
|
str(self.root),
|
|
"--inventory",
|
|
"docs/capability-inventory.json",
|
|
"--schema",
|
|
"docs/schemas/capability-inventory.schema.json",
|
|
"--mode",
|
|
mode,
|
|
]
|
|
if report:
|
|
# A previous case must not satisfy this invocation's report assertions.
|
|
report_path.unlink(missing_ok=True)
|
|
command.extend(["--report", str(report_path)])
|
|
completed = subprocess.run(
|
|
command,
|
|
check=False,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
cwd=str(ROOT),
|
|
)
|
|
return completed, report_path
|
|
|
|
def assert_rejected_in_both_modes(
|
|
self,
|
|
payload: dict[str, Any],
|
|
label: str,
|
|
*,
|
|
expect_named: tuple[str, ...] = (),
|
|
) -> None:
|
|
"""The agreed structural/semantic invalid row: exit 1, `status=invalid`, no traceback, no Gate B."""
|
|
for mode in ("validity", "feature-complete"):
|
|
with self.subTest(mutation=label, mode=mode):
|
|
result, _ = self.run_validator(payload, mode)
|
|
combined = result.stdout + result.stderr
|
|
self.assertNotIn("Traceback (most recent call last)", combined, combined)
|
|
self.assertNotIn(
|
|
"Traceback",
|
|
result.stderr,
|
|
f"{label}: a bounded diagnostic is required, never a traceback: {combined}",
|
|
)
|
|
self.assertEqual(
|
|
result.returncode,
|
|
1,
|
|
f"{label}: {REJECTED_BOTH_MODES} (exit 1), observed {result.returncode}: {combined}",
|
|
)
|
|
self.assertIn("status=invalid", combined, combined)
|
|
self.assertNotIn("status=complete", combined, combined)
|
|
self.assertNotIn("status=blocked", combined, combined)
|
|
self.assertNotRegex(combined, r"blockers=[1-9]")
|
|
for token in expect_named:
|
|
self.assertIn(
|
|
token,
|
|
combined,
|
|
f"{label}: the diagnostic must name {token!r}: {combined}",
|
|
)
|
|
|
|
# --- R1: structural Gate A -------------------------------------------------------------------
|
|
|
|
def test_control_reviewed_candidate_keeps_the_truthful_gate_verdicts(self) -> None:
|
|
"""Independent positive control: the unmutated reviewed candidate stays A0 / B3.
|
|
|
|
Every negative below differs from this document in exactly one factor, so a RED cannot be
|
|
explained by a broken harness or an already-invalid baseline.
|
|
"""
|
|
payload = self.candidate()
|
|
validity, _ = self.run_validator(payload, "validity")
|
|
self.assertEqual(validity.returncode, 0, validity.stdout + validity.stderr)
|
|
self.assertIn("Capability inventory validation passed", validity.stdout + validity.stderr)
|
|
self.assertRegex(validity.stdout + validity.stderr, r"\bprotocol_items=50\b")
|
|
|
|
feature_complete, _ = self.run_validator(payload, "feature-complete")
|
|
combined = feature_complete.stdout + feature_complete.stderr
|
|
self.assertEqual(feature_complete.returncode, 3, combined)
|
|
self.assertIn("status=blocked", combined)
|
|
# resources is a supported capability since Story 3.8, so the truthful window has 38
|
|
# (surface, revision) blockers instead of 40.
|
|
self.assertRegex(combined, r"blockers=38\b")
|
|
|
|
def test_r1_missing_product_is_rejected_in_both_modes(self) -> None:
|
|
payload = self.candidate()
|
|
del payload["product"]
|
|
self.assertNotIn("product", payload)
|
|
self.assert_rejected_in_both_modes(payload, "missing-product", expect_named=("product",))
|
|
|
|
def test_r1_non_hex_digest_is_rejected_in_both_modes(self) -> None:
|
|
payload = self.candidate()
|
|
self.revision_entry(payload, MODERN)["provenance"]["sha256"] = "not-a-digest"
|
|
self.assert_rejected_in_both_modes(payload, "bad-digest", expect_named=("sha256",))
|
|
|
|
def test_r1_non_date_capture_is_rejected_in_both_modes(self) -> None:
|
|
payload = self.candidate()
|
|
self.revision_entry(payload, MODERN)["provenance"]["captured_on"] = "not-a-date"
|
|
self.assert_rejected_in_both_modes(payload, "bad-date", expect_named=("captured_on",))
|
|
|
|
def test_r1_modern_revision_cannot_declare_a_session(self) -> None:
|
|
payload = self.candidate()
|
|
self.revision_entry(payload, MODERN)["session"] = True
|
|
self.assert_rejected_in_both_modes(payload, "wrong-modern-session", expect_named=(MODERN,))
|
|
|
|
def test_r1_reversed_ledger_order_is_rejected(self) -> None:
|
|
payload = self.candidate()
|
|
payload["protocol"]["revisions"] = list(reversed(payload["protocol"]["revisions"]))
|
|
self.assertEqual(
|
|
[entry["revision"] for entry in payload["protocol"]["revisions"]], [LEGACY, MODERN]
|
|
)
|
|
self.assert_rejected_in_both_modes(payload, "ledger-reversed")
|
|
|
|
def test_r1_duplicated_ledger_revision_is_rejected(self) -> None:
|
|
payload = self.candidate()
|
|
revisions = payload["protocol"]["revisions"]
|
|
payload["protocol"]["revisions"] = revisions + [copy.deepcopy(revisions[0])]
|
|
self.assertEqual(len(payload["protocol"]["revisions"]), 3)
|
|
self.assert_rejected_in_both_modes(payload, "ledger-duplicate")
|
|
|
|
def test_r1_non_string_kind_is_rejected_with_a_bounded_diagnostic(self) -> None:
|
|
payload = self.candidate()
|
|
self.tools_entry(payload)["kind"] = ["capability"]
|
|
self.assert_rejected_in_both_modes(payload, "kind-array", expect_named=("kind",))
|
|
|
|
def test_r1_non_string_subfeatures_are_rejected_with_a_bounded_diagnostic(self) -> None:
|
|
payload = self.candidate()
|
|
self.tools_entry(payload)["subfeatures"] = [{"bad": True}]
|
|
self.assert_rejected_in_both_modes(payload, "subfeatures-object", expect_named=("subfeatures",))
|
|
|
|
def test_r1_string_open_subfeatures_does_not_open_the_advertisement(self) -> None:
|
|
"""`open_subfeatures: "false"` is a wrong type and must not act as a truthy open-map flag."""
|
|
payload = self.candidate()
|
|
tools = self.tools_entry(payload)
|
|
tools["open_subfeatures"] = "false"
|
|
tools["advertised"] = {"tools": {"listChanged": False, "unexpected": True}}
|
|
self.assert_rejected_in_both_modes(payload, "string-open-map", expect_named=("tools",))
|
|
|
|
def test_r1_protocol_less_document_cannot_be_feature_complete(self) -> None:
|
|
"""A document with no `protocol` block cannot be reported as feature-complete.
|
|
|
|
Gate B is a v2/protocol scope gate: with no protocol block there is nothing that could be
|
|
complete, so `status=complete blockers=0` is a false claim (TL R1 / EH3 "v1 without protocol
|
|
returns B0 complete"). Validity mode for the legacy v1 flow-only document is left exactly as
|
|
accepted before (the existing V suite pins that behaviour); only the false feature-complete
|
|
verdict is the regression.
|
|
"""
|
|
payload = self.candidate()
|
|
payload["schema_version"] = 1
|
|
del payload["protocol"]
|
|
result, _ = self.run_validator(payload, "feature-complete")
|
|
combined = result.stdout + result.stderr
|
|
self.assertNotIn("status=complete", combined, combined)
|
|
self.assertNotRegex(combined, r"blockers=0\b", combined)
|
|
self.assertEqual(result.returncode, 1, combined)
|
|
self.assertIn("status=invalid", combined)
|
|
|
|
# --- R2: effective per-era validity/completeness ---------------------------------------------
|
|
|
|
def test_r2_unknown_per_revision_override_field_is_rejected(self) -> None:
|
|
payload = self.candidate()
|
|
self.tools_entry(payload)["per_revision"] = {MODERN: {"unknown": True}}
|
|
self.assert_rejected_in_both_modes(payload, "override-unknown-field", expect_named=("unknown",))
|
|
|
|
def test_r2_invalid_effective_status_is_rejected(self) -> None:
|
|
payload = self.candidate()
|
|
deferred = next(
|
|
cell for cell in payload["protocol"]["capabilities"] if cell["status"] == "deferred"
|
|
)
|
|
deferred.setdefault("per_revision", {})[MODERN] = {"status": "bogus"}
|
|
self.assert_rejected_in_both_modes(payload, "override-status-bogus", expect_named=("bogus",))
|
|
|
|
def test_r2_per_era_override_cannot_relabel_deferred_work_as_not_applicable(self) -> None:
|
|
"""TL R2: relabelling every applicable `deferred` surface per era yields a false B0.
|
|
|
|
The override is legal per `revisions[]` (no `OVERRIDE_OUTSIDE_REVISIONS`), so the bypass is
|
|
purely the missing *effective* per-era truthfulness/feature-complete evaluation.
|
|
"""
|
|
payload = self.candidate()
|
|
relabelled = 0
|
|
for cell in payload["protocol"]["capabilities"]:
|
|
if cell["status"] != "deferred":
|
|
continue
|
|
overrides = cell.setdefault("per_revision", {})
|
|
for era in cell["revisions"]:
|
|
overrides[era] = {"status": "not_applicable", "applicability": "stable"}
|
|
relabelled += 1
|
|
self.assertEqual(relabelled, 26, "the reviewed candidate defers 26 surfaces")
|
|
|
|
for mode in ("validity", "feature-complete"):
|
|
with self.subTest(mode=mode):
|
|
result, _ = self.run_validator(payload, mode)
|
|
combined = result.stdout + result.stderr
|
|
self.assertEqual(
|
|
result.returncode,
|
|
1,
|
|
"an effective per-era not_applicable that relabels applicable scheduled work "
|
|
f"must be invalid in BOTH modes (exit 1), observed {result.returncode}: {combined}",
|
|
)
|
|
self.assertIn("status=invalid", combined)
|
|
self.assertNotIn("status=complete", combined)
|
|
self.assertNotRegex(combined, r"blockers=0\b")
|
|
|
|
def test_r2_per_era_omission_of_a_reviewed_modern_surface_is_rejected(self) -> None:
|
|
"""Dropping the modern era from `tools.revisions[]` removes a reviewed modern surface.
|
|
|
|
Completeness is `eligibility-first` (rule O2): the applicable set of an era is the reviewed
|
|
era basis (`modern 35` / `legacy 43`), not the 50-identity default union, so the tool
|
|
declaration omitted from the modern era must be an `OMISSION` for that era.
|
|
"""
|
|
payload = self.candidate()
|
|
tools = self.tools_entry(payload)
|
|
self.assertEqual(sorted(tools["revisions"]), sorted([MODERN, LEGACY]))
|
|
tools["revisions"] = [LEGACY]
|
|
self.assert_rejected_in_both_modes(payload, "per-era-omission-tools", expect_named=("tools",))
|
|
|
|
# --- R3: malformed advertisement -------------------------------------------------------------
|
|
|
|
def test_r3_scalar_advertised_fragment_is_rejected(self) -> None:
|
|
payload = self.candidate()
|
|
self.tools_entry(payload)["advertised"] = {"tools": True}
|
|
self.assert_rejected_in_both_modes(payload, "advertised-scalar", expect_named=("tools",))
|
|
|
|
def test_r3_foreign_top_level_advertised_key_is_rejected(self) -> None:
|
|
payload = self.candidate()
|
|
self.tools_entry(payload)["advertised"] = {"prompts": {"listChanged": False}}
|
|
self.assert_rejected_in_both_modes(payload, "advertised-foreign-key", expect_named=("tools", "prompts"))
|
|
|
|
# --- R4: bounded valid JSON report -----------------------------------------------------------
|
|
|
|
def test_r4_large_valid_report_stays_parseable_json(self) -> None:
|
|
"""A valid 2048-character Unicode reason must not truncate the report into invalid JSON."""
|
|
payload = self.candidate()
|
|
for cell in payload["protocol"]["capabilities"]:
|
|
if cell["status"] == "deferred":
|
|
cell["reason"] = "я" * 2048
|
|
|
|
result, report_path = self.run_validator(payload, "feature-complete", report=True)
|
|
combined = result.stdout + result.stderr
|
|
self.assertIn("status=blocked", combined, combined)
|
|
self.assertTrue(report_path.exists(), f"the bounded JSON report must be written: {combined}")
|
|
|
|
raw = report_path.read_bytes()
|
|
self.assertLessEqual(len(raw), 65_536, "the report must stay within the byte cap")
|
|
try:
|
|
report = json.loads(raw)
|
|
except json.JSONDecodeError as error:
|
|
self.fail(
|
|
f"the bounded report must remain valid JSON, got {len(raw)} bytes: {error}; "
|
|
f"tail={raw[-64:]!r}"
|
|
)
|
|
self.assertEqual(report["status"], "blocked")
|
|
blockers = report["blockers"]
|
|
self.assertIsInstance(blockers, list)
|
|
|
|
# Honest disclosure: either the report carries every blocker, or it says so explicitly.
|
|
match = re.search(r"blockers=(\d+)", combined)
|
|
if match is None:
|
|
self.fail(f"the feature-complete summary must report the blocker count: {combined}")
|
|
total = int(match.group(1))
|
|
if len(blockers) < total:
|
|
disclosure = {"omitted", "omitted_blockers", "truncated", "truncation", "note"}
|
|
self.assertTrue(
|
|
disclosure & set(report),
|
|
f"a report that drops blockers must disclose the omission explicitly: {sorted(report)}",
|
|
)
|
|
|
|
# --- F1-F4 (TL run63 residual repair): the same oracles, now as permanent regressions ----------
|
|
|
|
def run_producer(
|
|
self,
|
|
payload: dict[str, Any],
|
|
revision: str = MODERN,
|
|
raw: str | None = None,
|
|
tag: str = "",
|
|
) -> tuple[subprocess.CompletedProcess[str], Path, Path]:
|
|
"""Run the standalone artifact producer on `payload` (real CLI, no server, no registry).
|
|
|
|
`raw` passes the on-disk bytes verbatim, so a non-standard JSON constant (NaN/Infinity) can be
|
|
fed to the bounded oracle without json.dumps normalising it away. `tag` namespaces the artifact
|
|
paths, so several producer cases (accepted and rejected) can share one temporary root without
|
|
reading each other's leftovers.
|
|
"""
|
|
inventory = self.root / "docs" / "capability-inventory.json"
|
|
if raw is None:
|
|
inventory.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
else:
|
|
inventory.write_text(raw, encoding="utf-8")
|
|
prefix = f"{tag}." if tag else ""
|
|
frozen = self.root / f"{prefix}{revision}-advertised-capabilities.frozen.json"
|
|
provenance = self.root / f"{prefix}{revision}-advertised-capabilities.provenance.json"
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(PRODUCER),
|
|
"--inventory",
|
|
str(inventory),
|
|
"--revision",
|
|
revision,
|
|
"--frozen",
|
|
str(frozen),
|
|
"--provenance",
|
|
str(provenance),
|
|
],
|
|
check=False,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
cwd=str(ROOT),
|
|
)
|
|
return completed, frozen, provenance
|
|
|
|
def test_f1_per_era_revisions_override_cannot_narrow_the_eligibility_scope(self) -> None:
|
|
"""F1: `per_revision[era].revisions = []` must not disable the declared applicability scope.
|
|
|
|
The reviewer reproduced a false readiness claim with exactly this one-factor mutation: every
|
|
truthfully deferred surface kept its `deferred` default but lost its era scope, so Gate B
|
|
reported `status=complete blockers=0` while the canonical window still has 40 blockers.
|
|
"""
|
|
payload = self.candidate()
|
|
narrowed = 0
|
|
for cell in payload["protocol"]["capabilities"]:
|
|
if cell["status"] != "deferred":
|
|
continue
|
|
overrides = cell.setdefault("per_revision", {})
|
|
for era in cell["revisions"]:
|
|
# Merge, never replace: the reviewed per-era overrides must survive the mutation, so
|
|
# the only difference from the positive control is the narrowed applicability scope.
|
|
overrides[era] = {**overrides.get(era, {}), "revisions": []}
|
|
narrowed += 1
|
|
self.assertEqual(narrowed, 26, "the reviewed candidate defers 26 surfaces")
|
|
|
|
self.assert_rejected_in_both_modes(payload, "per-era-revisions-narrowing", expect_named=("revisions",))
|
|
|
|
def test_f2_effective_per_era_advertisement_is_validated(self) -> None:
|
|
"""F2: an advertisement that only becomes wrong after the O2 merge is a Gate A error.
|
|
|
|
Two one-factor mutations of the supported server `tools` capability: a scalar fragment and a
|
|
`null` fragment for one era. The standalone producer already rejects both, so a validator that
|
|
accepted them contradicted the artifact oracle it must agree with.
|
|
"""
|
|
scalar = self.candidate()
|
|
self.tools_entry(scalar)["per_revision"] = {MODERN: {"advertised": {"tools": True}}}
|
|
self.assert_rejected_in_both_modes(scalar, "effective-scalar-advertised", expect_named=("tools",))
|
|
|
|
null_fragment = self.candidate()
|
|
self.tools_entry(null_fragment)["per_revision"] = {MODERN: {"advertised": None}}
|
|
self.assert_rejected_in_both_modes(
|
|
null_fragment, "effective-null-advertised", expect_named=("advertised",)
|
|
)
|
|
|
|
for label, payload in (("effective-scalar-advertised", scalar), ("effective-null-advertised", null_fragment)):
|
|
with self.subTest(mutation=label, producer=MODERN):
|
|
completed, frozen, _ = self.run_producer(payload, MODERN)
|
|
combined = completed.stdout + completed.stderr
|
|
self.assertNotIn("Traceback", completed.stderr, combined)
|
|
self.assertEqual(
|
|
completed.returncode,
|
|
1,
|
|
f"{label}: the artifact producer must reject the same document (exit 1): {combined}",
|
|
)
|
|
self.assertIn("PROJECTION_REJECTED", combined, combined)
|
|
self.assertFalse(frozen.exists(), f"{label}: a rejected projection must write no frozen file")
|
|
|
|
def test_f3_missing_revision_contract_fields_are_rejected(self) -> None:
|
|
"""F3: the reviewed role/lifecycle/required-method contract of a revision is mandatory."""
|
|
for field in ("role", "lifecycle", "required_methods"):
|
|
payload = self.candidate()
|
|
del self.revision_entry(payload, MODERN)[field]
|
|
with self.subTest(missing=field):
|
|
self.assert_rejected_in_both_modes(payload, f"missing-revision-{field}", expect_named=(field,))
|
|
|
|
def test_f3_foreign_product_identity_is_rejected(self) -> None:
|
|
"""F3: the canonical product identity is a constant, not free text (schema `const`)."""
|
|
payload = self.candidate()
|
|
payload["product"] = "other-product"
|
|
self.assert_rejected_in_both_modes(
|
|
payload, "foreign-product", expect_named=("crank-community", "other-product")
|
|
)
|
|
|
|
def test_f4_invalid_report_is_byte_capped_with_a_disclosed_omission(self) -> None:
|
|
"""F4: the invalid-mode JSON report must honour the 65536-byte cap and say what it dropped.
|
|
|
|
The reviewer measured 115118/115126-byte reports for a document with 1000 unknown top-level
|
|
keys: the byte cap was announced but not enforced, because the whole diagnostics array was
|
|
kept. The bounded report must stay valid JSON, stay within the cap and disclose the omission.
|
|
"""
|
|
payload = self.candidate()
|
|
for index in range(1000):
|
|
payload[f"unknown_field_{index:04d}"] = "x" * 230
|
|
|
|
for mode in ("validity", "feature-complete"):
|
|
with self.subTest(mode=mode):
|
|
result, report_path = self.run_validator(payload, mode, report=True)
|
|
combined = result.stdout + result.stderr
|
|
self.assertEqual(result.returncode, 1, combined)
|
|
self.assertIn("status=invalid", combined, combined)
|
|
self.assertTrue(report_path.exists(), combined)
|
|
|
|
raw = report_path.read_bytes()
|
|
self.assertLessEqual(
|
|
len(raw), 65_536, f"the invalid-mode report must stay within the byte cap: {len(raw)} bytes"
|
|
)
|
|
try:
|
|
report = json.loads(raw)
|
|
except json.JSONDecodeError as error:
|
|
self.fail(f"the capped report must stay valid JSON ({len(raw)} bytes): {error}")
|
|
self.assertEqual(report["status"], "invalid")
|
|
self.assertTrue(report.get("truncated"), f"the omission must be disclosed: {sorted(report)}")
|
|
self.assertGreater(
|
|
int(report.get("omitted_diagnostics", 0)),
|
|
0,
|
|
f"the number of omitted diagnostics must be named: {sorted(report)}",
|
|
)
|
|
self.assertIsInstance(report.get("diagnostics"), list)
|
|
|
|
def test_f8_standalone_producer_rejects_malformed_input_without_a_traceback(self) -> None:
|
|
"""F8: the artifact CLI is a bounded oracle, its negative verdict is a diagnostic, not a crash."""
|
|
cases: list[tuple[str, Any, Any]] = [
|
|
("kind-array", "kind", ["capability"]),
|
|
("subfeatures-objects", "subfeatures", [{"bad": True}]),
|
|
("open-subfeatures-string", "open_subfeatures", "false"),
|
|
]
|
|
for label, field, value in cases:
|
|
payload = self.candidate()
|
|
tools = self.tools_entry(payload)
|
|
tools[field] = value
|
|
if label == "open-subfeatures-string":
|
|
tools["advertised"] = {"tools": {"listChanged": False, "unexpected": True}}
|
|
with self.subTest(mutation=label, revision=MODERN):
|
|
completed, frozen, _ = self.run_producer(payload, MODERN)
|
|
combined = completed.stdout + completed.stderr
|
|
self.assertNotIn(
|
|
"Traceback (most recent call last)",
|
|
combined,
|
|
f"{label}: a malformed input must produce a bounded diagnostic, not a traceback: {combined}",
|
|
)
|
|
self.assertEqual(completed.returncode, 1, f"{label}: {combined}")
|
|
self.assertIn("PROJECTION_REJECTED", combined, combined)
|
|
self.assertFalse(frozen.exists(), f"{label}: nothing may be written for a rejected input")
|
|
|
|
# --- G1-G5 (TL run65 residual repair): the same oracles as permanent regressions ---------------
|
|
|
|
def test_g1_per_era_applicability_cannot_relabel_deferred_work(self) -> None:
|
|
"""G1: `per_revision[era].applicability = "deprecated"` must not remove the reviewed deferral.
|
|
|
|
The reviewer reproduced the F1 false-readiness claim through a second one-factor mutation:
|
|
every truthfully deferred surface kept `status=deferred` but declared its effective
|
|
applicability `deprecated` for every era, and Gate B reported `status=complete blockers=0`
|
|
while the canonical window still has 40 blockers. Applicability is a reviewed statement about
|
|
the pinned revision (`surface-inventory.md` section 8): only the six reviewed modern-window
|
|
deprecations may carry that value, and legacy carries no `@deprecated` marker at all.
|
|
"""
|
|
payload = self.candidate()
|
|
mutated = 0
|
|
for cell in payload["protocol"]["capabilities"]:
|
|
if cell["status"] != "deferred":
|
|
continue
|
|
overrides = cell.setdefault("per_revision", {})
|
|
for era in cell["revisions"]:
|
|
# Merge, never replace: the reviewed per-era overrides survive, so the applicability
|
|
# value is the only factor that changes.
|
|
overrides[era] = {**overrides.get(era, {}), "applicability": "deprecated"}
|
|
mutated += 1
|
|
self.assertEqual(mutated, 26, "the reviewed candidate defers 26 surfaces")
|
|
|
|
self.assert_rejected_in_both_modes(
|
|
payload, "per-era-applicability-deprecated", expect_named=("applicability",)
|
|
)
|
|
|
|
def test_g1_only_the_reviewed_modern_deprecations_may_be_deprecated(self) -> None:
|
|
"""G1 negative of the negative: the committed reviewed deprecations stay valid (control).
|
|
|
|
The rule must not forbid every applicability override: `surface-inventory.md` section 8 pins
|
|
exactly six modern-window `@deprecated` surfaces (logging, notifications/message, roots,
|
|
roots/list, sampling, sampling/createMessage) and the reviewed candidate already carries them.
|
|
"""
|
|
payload = self.candidate()
|
|
deprecated = {
|
|
(cell["role"], cell["kind"], cell["surface"])
|
|
for cell in payload["protocol"]["capabilities"]
|
|
if any(
|
|
override.get("applicability") == "deprecated"
|
|
for override in (cell.get("per_revision") or {}).values()
|
|
)
|
|
}
|
|
self.assertEqual(
|
|
deprecated,
|
|
{
|
|
("server", "capability", "logging"),
|
|
("server", "notification", "notifications/message"),
|
|
("client", "capability", "roots"),
|
|
("server", "method", "roots/list"),
|
|
("client", "capability", "sampling"),
|
|
("server", "method", "sampling/createMessage"),
|
|
},
|
|
"the reviewed modern-window deprecations of surface-inventory.md section 8",
|
|
)
|
|
result, _ = self.run_validator(payload, "validity")
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
|
|
# And the deprecated set may not leak into the legacy window, which carries no marker.
|
|
leaked = self.candidate()
|
|
for cell in leaked["protocol"]["capabilities"]:
|
|
if cell["status"] != "deferred" or LEGACY not in cell["revisions"]:
|
|
continue
|
|
overrides = cell.setdefault("per_revision", {})
|
|
overrides[LEGACY] = {**overrides.get(LEGACY, {}), "applicability": "deprecated"}
|
|
self.assert_rejected_in_both_modes(
|
|
leaked, "legacy-applicability-deprecated", expect_named=("applicability",)
|
|
)
|
|
|
|
def test_g3_structural_types_and_bounds_cover_every_protocol_record(self) -> None:
|
|
"""G3: the schema-shaped type/bound checks apply to each record, advertised or not.
|
|
|
|
The reviewer reproduced five one-factor mutations that passed Gate A because the type checks
|
|
only ran inside the advertisement branch (or not at all): `subfeatures`/`open_subfeatures` on a
|
|
non-advertised record, a `null` `per_revision`, a duplicated revision and an `owner`
|
|
(`evidence_target`) longer than the pinned 256-character bound. Applicability of an
|
|
advertisement never suspends the type contract of the document itself.
|
|
"""
|
|
|
|
subfeatures_null = self.candidate()
|
|
next(
|
|
cell for cell in subfeatures_null["protocol"]["capabilities"] if cell["surface"] == "logging"
|
|
)["subfeatures"] = None
|
|
open_string = self.candidate()
|
|
next(
|
|
cell for cell in open_string["protocol"]["capabilities"] if cell["surface"] == "logging"
|
|
)["open_subfeatures"] = "false"
|
|
per_revision_null = self.candidate()
|
|
self.tools_entry(per_revision_null)["per_revision"] = None
|
|
duplicate_revision = self.candidate()
|
|
self.tools_entry(duplicate_revision)["revisions"] = [MODERN, LEGACY, MODERN]
|
|
owner_257 = self.candidate()
|
|
self.tools_entry(owner_257)["owner"] = "o" * 257
|
|
evidence_257 = self.candidate()
|
|
self.tools_entry(evidence_257)["evidence_target"] = "e" * 257
|
|
effective_null = self.candidate()
|
|
deferred_entry = next(
|
|
cell for cell in effective_null["protocol"]["capabilities"] if cell["surface"] == "logging"
|
|
)
|
|
deferred_entry["per_revision"][MODERN] = {
|
|
**deferred_entry["per_revision"].get(MODERN, {}),
|
|
"subfeatures": None,
|
|
}
|
|
|
|
for label, candidate_payload, token in (
|
|
("default-subfeatures-null", subfeatures_null, "subfeatures"),
|
|
("default-open-subfeatures-string", open_string, "open_subfeatures"),
|
|
("default-per-revision-null", per_revision_null, "per_revision"),
|
|
("default-duplicate-revision", duplicate_revision, "revisions"),
|
|
("default-owner-over-256", owner_257, "owner"),
|
|
("default-evidence-target-over-256", evidence_257, "evidence_target"),
|
|
("effective-subfeatures-null", effective_null, "subfeatures"),
|
|
):
|
|
with self.subTest(mutation=label):
|
|
self.assert_rejected_in_both_modes(candidate_payload, label, expect_named=(token,))
|
|
|
|
def test_g4_advertisement_leaf_keeps_its_pinned_wire_type(self) -> None:
|
|
"""G4: `tools.listChanged` is a boolean in the pinned revision schema, not any JSON value.
|
|
|
|
`reviewed-inputs/2026-07-28-schema.ts` types `ServerCapabilities.tools.listChanged?: boolean`.
|
|
An array leaf passed Gate A and was written into the frozen artifact, so both executable
|
|
consumers accepted a fragment the pinned type does not allow.
|
|
"""
|
|
payload = self.candidate()
|
|
self.tools_entry(payload)["advertised"] = {"tools": {"listChanged": []}}
|
|
self.assert_rejected_in_both_modes(payload, "listChanged-array", expect_named=("listChanged",))
|
|
|
|
completed, frozen, _ = self.run_producer(payload, MODERN)
|
|
combined = completed.stdout + completed.stderr
|
|
self.assertEqual(completed.returncode, 1, combined)
|
|
self.assertIn("PROJECTION_REJECTED", combined, combined)
|
|
self.assertFalse(frozen.exists(), "a rejected projection must write no frozen file")
|
|
|
|
control = self.candidate()
|
|
self.tools_entry(control)["advertised"] = {"tools": {"listChanged": True}}
|
|
accepted, frozen, _ = self.run_producer(control, MODERN)
|
|
self.assertEqual(accepted.returncode, 0, accepted.stdout + accepted.stderr)
|
|
self.assertEqual(
|
|
frozen.read_text(encoding="utf-8"),
|
|
'{"resources":{"listChanged":false,"subscribe":false},"tools":{"listChanged":true}}',
|
|
)
|
|
|
|
def test_g4_producer_writes_only_standard_json(self) -> None:
|
|
"""G4: a non-standard JSON constant may never become a successful frozen artifact.
|
|
|
|
`json.dumps` emits `NaN`/`Infinity` by default. The reviewer fed a `NaN` leaf through the CLI
|
|
and got exit 0 plus a frozen file no standard JSON parser accepts.
|
|
"""
|
|
text = json.dumps(self.candidate(), indent=2, sort_keys=True).replace(
|
|
'"listChanged": false', '"listChanged": NaN'
|
|
)
|
|
self.assertIn("NaN", text)
|
|
completed, frozen, provenance = self.run_producer({}, MODERN, raw=text)
|
|
combined = completed.stdout + completed.stderr
|
|
self.assertNotIn("Traceback (most recent call last)", combined, combined)
|
|
self.assertEqual(completed.returncode, 1, combined)
|
|
self.assertIn("PROJECTION_REJECTED", combined, combined)
|
|
self.assertFalse(frozen.exists(), "a rejected projection must write no frozen file")
|
|
self.assertFalse(provenance.exists(), "a rejected projection must write no provenance sidecar")
|
|
|
|
def test_g5_standalone_producer_rejects_a_structural_scope_override(self) -> None:
|
|
"""G5: the artifact CLI applies the same rule O1 as the validator, on its own.
|
|
|
|
With `per_revision[era].revisions = []` on every entry the validator already answered
|
|
`OVERRIDE_NOT_ALLOWED_FIELD` (exit 1) while the standalone producer exited 0 and wrote an empty
|
|
frozen artifact: the forbidden override narrowed the eligibility scope before the O2 merge.
|
|
A rejected projection writes neither the frozen payload nor the provenance sidecar.
|
|
"""
|
|
payload = self.candidate()
|
|
for cell in payload["protocol"]["capabilities"]:
|
|
overrides = cell.setdefault("per_revision", {})
|
|
for era in cell["revisions"]:
|
|
overrides[era] = {**overrides.get(era, {}), "revisions": []}
|
|
|
|
self.assert_rejected_in_both_modes(payload, "producer-structural-override", expect_named=("revisions",))
|
|
|
|
for revision in (MODERN, LEGACY):
|
|
with self.subTest(revision=revision, producer="structural-override"):
|
|
completed, frozen, provenance = self.run_producer(payload, revision)
|
|
combined = completed.stdout + completed.stderr
|
|
self.assertNotIn("Traceback (most recent call last)", combined, combined)
|
|
self.assertEqual(completed.returncode, 1, combined)
|
|
self.assertIn("PROJECTION_REJECTED", combined, combined)
|
|
self.assertFalse(frozen.exists(), "a rejected projection writes no frozen artifact")
|
|
self.assertFalse(provenance.exists(), "a rejected projection writes no provenance")
|
|
|
|
|
|
# --- H1-H2 (TL run67 residual repair): the same oracles as permanent regressions ---------------
|
|
|
|
def test_h1_wrong_typed_capability_revision_member_is_a_bounded_negative(self) -> None:
|
|
"""H1: `tools.revisions = [{}]` / `[[]]` is a bounded type error, never a traceback.
|
|
|
|
The reviewer reproduced four validator subprocesses that crashed with `TypeError` (the member
|
|
was used by `set(revisions)` before its type was checked), so the CLI exited 1 without
|
|
`status=invalid` and without any report. The member type is checked before any
|
|
set/hash/membership use.
|
|
"""
|
|
for label, member in (("revision-object", {}), ("revision-array", [])):
|
|
payload = self.candidate()
|
|
self.tools_entry(payload)["revisions"] = [member]
|
|
with self.subTest(mutation=label):
|
|
self.assert_rejected_in_both_modes(payload, label, expect_named=("revisions",))
|
|
|
|
def test_h1_unknown_nested_ledger_provenance_field_is_rejected(self) -> None:
|
|
"""H1: `provenance` is a closed object (`additionalProperties: false`), not an open bag.
|
|
|
|
`protocol.revisions[0].provenance.unexpected = true` was accepted as Gate A / B3; an unknown
|
|
nested key of the pinned ledger is a structural error like any unknown top-level key.
|
|
"""
|
|
payload = self.candidate()
|
|
self.revision_entry(payload, MODERN)["provenance"]["unexpected"] = True
|
|
self.assert_rejected_in_both_modes(
|
|
payload, "provenance-unknown-field", expect_named=("unexpected",)
|
|
)
|
|
|
|
def test_h1_present_reason_keeps_its_pinned_type_and_bound(self) -> None:
|
|
"""H1: `reason` is optional in *presence*, never in type.
|
|
|
|
`supported tools.reason = []` (a present container) and a 2049-character string both passed
|
|
Gate A / B3. The rule is applied to the default record and to the effective per-era record,
|
|
because `reason` is one of the rule O1 override candidates.
|
|
"""
|
|
for label, value in (("reason-array", []), ("reason-over-2048", "r" * 2_049)):
|
|
payload = self.candidate()
|
|
self.tools_entry(payload)["reason"] = value
|
|
with self.subTest(mutation=label):
|
|
self.assert_rejected_in_both_modes(payload, label, expect_named=("reason",))
|
|
|
|
effective = self.candidate()
|
|
self.tools_entry(effective)["per_revision"] = {MODERN: {"reason": []}}
|
|
self.assert_rejected_in_both_modes(
|
|
effective, "effective-reason-array", expect_named=("reason",)
|
|
)
|
|
|
|
def test_h1_capability_revisions_keep_the_fixed_window_order(self) -> None:
|
|
"""H1/O3: `revisions[]` order is pinned to `[2026-07-28, 2025-11-25]`.
|
|
|
|
`tools.revisions = [legacy, modern]` keeps the member set and was accepted; only the order
|
|
diagnostic catches it. The legal one-era scopes of the unmutated candidate (server/client
|
|
`extensions` modern-only, server/client `tasks` legacy-only) remain valid, which is the
|
|
positive control of the fix.
|
|
"""
|
|
payload = self.candidate()
|
|
tools = self.tools_entry(payload)
|
|
self.assertEqual(tools["revisions"], [MODERN, LEGACY])
|
|
tools["revisions"] = [LEGACY, MODERN]
|
|
self.assert_rejected_in_both_modes(
|
|
payload, "capability-revisions-reversed", expect_named=("revisions",)
|
|
)
|
|
|
|
control = self.candidate()
|
|
single_era = {
|
|
(cell["role"], cell["kind"], cell["surface"]): cell["revisions"]
|
|
for cell in control["protocol"]["capabilities"]
|
|
if len(cell["revisions"]) == 1
|
|
}
|
|
self.assertEqual(single_era[("server", "capability", "extensions")], [MODERN])
|
|
self.assertEqual(single_era[("client", "capability", "tasks")], [LEGACY])
|
|
result, _ = self.run_validator(control, "validity")
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
|
|
def test_h1_present_and_required_field_table_of_the_ledger_and_capability_records(self) -> None:
|
|
"""H1: one bounded negative table over the current ledger and capability records.
|
|
|
|
Every row is a single-factor mutation of the reviewed candidate - a missing required field, a
|
|
`null`/scalar/container of the wrong type, an unknown nested key, a wrong enum value or an
|
|
over-long value - and must be `invalid1` in both modes with a bounded named diagnostic. The
|
|
table closes the present/required field contract of the current protocol ledger and of the
|
|
default/effective capability record as a whole instead of repairing one adjacent bypass at a
|
|
time (the schema `additionalProperties: false`, type, enum, required and limit rules).
|
|
"""
|
|
|
|
def ledger(field: str, value: Any) -> Any:
|
|
def apply(payload: dict[str, Any]) -> None:
|
|
self.revision_entry(payload, MODERN)[field] = value
|
|
|
|
return apply
|
|
|
|
def provenance(field: str, value: Any) -> Any:
|
|
def apply(payload: dict[str, Any]) -> None:
|
|
self.revision_entry(payload, MODERN)["provenance"][field] = value
|
|
|
|
return apply
|
|
|
|
def capability(field: str, value: Any) -> Any:
|
|
def apply(payload: dict[str, Any]) -> None:
|
|
self.tools_entry(payload)[field] = value
|
|
|
|
return apply
|
|
|
|
def effective(field: str, value: Any) -> Any:
|
|
def apply(payload: dict[str, Any]) -> None:
|
|
entry = self.tools_entry(payload)
|
|
overrides = entry.setdefault("per_revision", {})
|
|
overrides[MODERN] = {**overrides.get(MODERN, {}), field: value}
|
|
|
|
return apply
|
|
|
|
rows: list[tuple[str, Any, tuple[str, ...]]] = [
|
|
("ledger-revision-null", ledger("revision", None), ("revision",)),
|
|
("ledger-role-scalar", ledger("role", 7), ("role",)),
|
|
("ledger-lifecycle-container", ledger("lifecycle", []), ("lifecycle",)),
|
|
("ledger-required-methods-scalar", ledger("required_methods", "server/discover"), ("required_methods",)),
|
|
("ledger-session-wrong-type", ledger("session", "false"), ("session",)),
|
|
("ledger-provenance-locator-null", provenance("locator", None), ("locator",)),
|
|
("ledger-provenance-digest-scalar", provenance("sha256", 42), ("sha256",)),
|
|
("ledger-provenance-date-format", provenance("captured_on", "2026-10-11T00:00:00Z"), ("captured_on",)),
|
|
("ledger-provenance-unknown-key", provenance("unexpected", True), ("unexpected",)),
|
|
("capability-surface-scalar", capability("surface", 42), ("surface",)),
|
|
("capability-kind-null", capability("kind", None), ("kind",)),
|
|
("capability-role-unknown-enum", capability("role", "serverr"), ("role",)),
|
|
("capability-status-container", capability("status", ["supported"]), ("status",)),
|
|
("capability-applicability-null", capability("applicability", None), ("applicability",)),
|
|
("capability-revisions-null", capability("revisions", None), ("revisions",)),
|
|
("capability-revisions-empty", capability("revisions", []), ("revisions",)),
|
|
("capability-revisions-object-member", capability("revisions", [{}]), ("revisions",)),
|
|
("capability-revisions-scalar", capability("revisions", MODERN), ("revisions",)),
|
|
("capability-revisions-reversed", capability("revisions", [LEGACY, MODERN]), ("revisions",)),
|
|
("capability-subfeatures-scalar", capability("subfeatures", "listChanged"), ("subfeatures",)),
|
|
("capability-open-subfeatures-string", capability("open_subfeatures", "false"), ("open_subfeatures",)),
|
|
("capability-per-revision-container", capability("per_revision", []), ("per_revision",)),
|
|
("capability-owner-null", capability("owner", None), ("owner",)),
|
|
("capability-fallback-over-2048", capability("fallback", "f" * 2_049), ("fallback",)),
|
|
("capability-evidence-target-container", capability("evidence_target", []), ("evidence_target",)),
|
|
("capability-requirement-foreign", capability("requirement", "not-an-fr"), ("requirement",)),
|
|
("capability-reason-array", capability("reason", []), ("reason",)),
|
|
("capability-advertised-list", capability("advertised", []), ("advertised",)),
|
|
("effective-status-unknown-enum", effective("status", "bogus"), ("status",)),
|
|
("effective-owner-null", effective("owner", None), ("owner",)),
|
|
("effective-reason-array", effective("reason", []), ("reason",)),
|
|
("effective-advertised-null", effective("advertised", None), ("advertised",)),
|
|
("effective-open-subfeatures-string", effective("open_subfeatures", "false"), ("open_subfeatures",)),
|
|
("effective-subfeatures-scalar", effective("subfeatures", "listChanged"), ("subfeatures",)),
|
|
]
|
|
|
|
for label, mutate, tokens in rows:
|
|
payload = self.candidate()
|
|
mutate(payload)
|
|
with self.subTest(mutation=label):
|
|
self.assert_rejected_in_both_modes(payload, label, expect_named=tokens)
|
|
|
|
def test_h2_declared_subfeature_outside_the_reviewed_basis_is_rejected(self) -> None:
|
|
"""H2: `subfeatures` is the observed pinned sub-key list, not a self-declaration.
|
|
|
|
The reviewer declared an invented sub-key on the supported server `tools` capability
|
|
(`subfeatures = ["invented"]`): Gate A accepted it and the standalone producer exited 0 for both
|
|
eras. The declaration is compared with the reviewed per-era accounting
|
|
(`surface-inventory.md` sections 3-4: `tools` declares exactly `listChanged` in both eras).
|
|
"""
|
|
payload = self.candidate()
|
|
self.tools_entry(payload)["subfeatures"] = ["invented"]
|
|
self.assert_rejected_in_both_modes(
|
|
payload, "unknown-subfeature-declaration", expect_named=("invented",)
|
|
)
|
|
|
|
for revision in (MODERN, LEGACY):
|
|
with self.subTest(mutation="unknown-subfeature-declaration", revision=revision):
|
|
completed, frozen, provenance = self.run_producer(payload, revision)
|
|
combined = completed.stdout + completed.stderr
|
|
self.assertNotIn("Traceback (most recent call last)", combined, combined)
|
|
self.assertEqual(completed.returncode, 1, combined)
|
|
self.assertIn("PROJECTION_REJECTED", combined, combined)
|
|
self.assertFalse(frozen.exists(), "a rejected projection writes no frozen artifact")
|
|
self.assertFalse(provenance.exists(), "a rejected projection writes no provenance")
|
|
|
|
def test_h2_invented_tools_leaf_cannot_be_legitimized_by_its_own_declaration(self) -> None:
|
|
"""H2 composite bypass: declaring the invented key *and* advertising it is still not pinned.
|
|
|
|
`subfeatures = ["invented"]`, `open_subfeatures = false` and `advertised =
|
|
{"tools": {"invented": true}}` together produced validator A0/B3 and a frozen payload
|
|
`{"tools":{"invented":true}}` for both eras. No combination of the declaration and the payload
|
|
may legitimize a leaf the reviewed basis does not declare.
|
|
"""
|
|
payload = self.candidate()
|
|
tools = self.tools_entry(payload)
|
|
tools["subfeatures"] = ["invented"]
|
|
tools["open_subfeatures"] = False
|
|
tools["advertised"] = {"tools": {"invented": True}}
|
|
self.assert_rejected_in_both_modes(
|
|
payload, "invented-tools-leaf-bypass", expect_named=("invented",)
|
|
)
|
|
|
|
for revision in (MODERN, LEGACY):
|
|
with self.subTest(mutation="invented-tools-leaf-bypass", revision=revision):
|
|
completed, frozen, provenance = self.run_producer(payload, revision)
|
|
combined = completed.stdout + completed.stderr
|
|
self.assertNotIn("Traceback (most recent call last)", combined, combined)
|
|
self.assertEqual(completed.returncode, 1, combined)
|
|
self.assertIn("PROJECTION_REJECTED", combined, combined)
|
|
self.assertFalse(frozen.exists(), "a rejected projection writes no frozen artifact")
|
|
self.assertFalse(provenance.exists(), "a rejected projection writes no provenance")
|
|
|
|
def test_h2_per_era_subfeature_basis_is_pinned(self) -> None:
|
|
"""H2 control/negatives: the reviewed open maps stay legal, the pinned era difference is exact.
|
|
|
|
The truthful declared accounting of the unmutated candidate stays valid (`experimental` and
|
|
`extensions` are the reviewed open maps, client `roots` is empty in the modern window and
|
|
`listChanged` in the legacy one, the two `tasks` rows declare different sub-keys per side), so
|
|
the bounded comparison does not forbid real open maps for the sake of one `tools` case.
|
|
"""
|
|
control = self.candidate()
|
|
open_maps = {
|
|
(cell["role"], cell["surface"]): cell["open_subfeatures"]
|
|
for cell in control["protocol"]["capabilities"]
|
|
if cell["kind"] == "capability" and cell["open_subfeatures"]
|
|
}
|
|
self.assertEqual(
|
|
open_maps,
|
|
{
|
|
("server", "experimental"): True,
|
|
("server", "extensions"): True,
|
|
("client", "experimental"): True,
|
|
("client", "extensions"): True,
|
|
},
|
|
"the reviewed open maps of surface-inventory.md sections 3-4",
|
|
)
|
|
result, _ = self.run_validator(control, "validity")
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
|
|
# modern `roots` is the pinned empty object: the legacy-only `listChanged` may not leak into it.
|
|
leaked = self.candidate()
|
|
roots = next(
|
|
cell for cell in leaked["protocol"]["capabilities"] if cell["surface"] == "roots"
|
|
)
|
|
overrides = roots.setdefault("per_revision", {})
|
|
overrides[MODERN] = {**overrides.get(MODERN, {}), "subfeatures": ["listChanged"]}
|
|
self.assert_rejected_in_both_modes(
|
|
leaked, "modern-roots-listChanged", expect_named=("listChanged",)
|
|
)
|
|
|
|
# The client `tasks` row declares the client-side sub-keys, not the server-side ones.
|
|
foreign = self.candidate()
|
|
client_tasks = next(
|
|
cell
|
|
for cell in foreign["protocol"]["capabilities"]
|
|
if cell["surface"] == "tasks" and cell["role"] == "client"
|
|
)
|
|
client_tasks["subfeatures"] = ["list", "cancel", "requests.tools.call"]
|
|
self.assert_rejected_in_both_modes(
|
|
foreign, "client-tasks-server-subkeys", expect_named=("requests.tools.call",)
|
|
)
|
|
|
|
# --- J1-J2 (TL run69 residual repair): the same oracles as permanent regressions ---------------
|
|
|
|
def test_j1_long_revision_member_list_is_a_bounded_negative(self) -> None:
|
|
"""J1: the revision-member type is checked over the whole array, not only over a prefix window.
|
|
|
|
`tools.revisions = [modern] * 257 + [{}]` (and an array member in the tail): every member of the
|
|
bounded prefix was well formed, so the malformed tail escaped the member scan and the whole
|
|
array reached `set(revisions)` - the CLI exited 1 with a `TypeError` traceback, printed no
|
|
`status=invalid` and wrote no report. The `revision-window-edge` row is the last index the old
|
|
prefix covered; the two `tail` rows are one index past it. Every row must be rejected by the
|
|
named member diagnostic with a parseable bounded report.
|
|
"""
|
|
rows = (
|
|
("revision-window-edge-object", [MODERN] * 256 + [{}]),
|
|
("revision-window-tail-object", [MODERN] * 257 + [{}]),
|
|
("revision-window-tail-array", [MODERN] * 257 + [[]]),
|
|
)
|
|
for label, revisions in rows:
|
|
payload = self.candidate()
|
|
self.tools_entry(payload)["revisions"] = revisions
|
|
tools_index = payload["protocol"]["capabilities"].index(self.tools_entry(payload))
|
|
expected_pointer = f"/protocol/capabilities/{tools_index}/revisions/{len(revisions) - 1}"
|
|
with self.subTest(mutation=label):
|
|
self.assert_rejected_in_both_modes(payload, label, expect_named=("revisions",))
|
|
for mode in ("validity", "feature-complete"):
|
|
with self.subTest(mutation=label, mode=mode, probe="report"):
|
|
result, report = self.run_validator(payload, mode, report=True)
|
|
combined = result.stdout + result.stderr
|
|
self.assertNotIn("Traceback", combined, combined)
|
|
self.assertEqual(result.returncode, 1, combined)
|
|
self.assertTrue(
|
|
report.is_file(), "a rejected document still writes its bounded report"
|
|
)
|
|
report_bytes = report.read_bytes()
|
|
self.assertLessEqual(len(report_bytes), 65_536, "the report is byte-capped")
|
|
document = json.loads(report_bytes.decode("utf-8"))
|
|
self.assertEqual(document["mode"], mode)
|
|
self.assertEqual(document["status"], "invalid")
|
|
diagnostics = [
|
|
(item["code"], item["pointer"]) for item in document["diagnostics"]
|
|
]
|
|
self.assertIn(
|
|
("INVALID_FORMAT", expected_pointer),
|
|
diagnostics,
|
|
f"the current malformed revision member must be named: {diagnostics}",
|
|
)
|
|
|
|
def test_j2_producer_rejects_wrong_typed_accounting_of_a_non_projected_row(self) -> None:
|
|
"""J2: the standalone producer refuses a wrong-typed `subfeatures`/`open_subfeatures` as well.
|
|
|
|
The reviewer reproduced six false producer successes: `client sampling.subfeatures = null`,
|
|
`... = [{}]` and `open_subfeatures = "false"` were rejected by the validator (`invalid1`), while
|
|
the producer exited 0 and wrote the frozen payload plus the provenance sidecar for both eras.
|
|
The same path covered a truthfully deferred known capability and a field declared only in one
|
|
era's `per_revision` override (`subfeatures = null` for the modern era: only that era may be
|
|
rejected, the legal effective record of the other era stays projectable). The reviewed per-era
|
|
basis is now type-checked before it is compared, so a wrong-typed declaration of a capability
|
|
record is a bounded rejection instead of a silently skipped row.
|
|
"""
|
|
|
|
def client_sampling(payload: dict[str, Any]) -> dict[str, Any]:
|
|
return next(
|
|
cell
|
|
for cell in payload["protocol"]["capabilities"]
|
|
if cell["kind"] == "capability"
|
|
and cell["role"] == "client"
|
|
and cell["surface"] == "sampling"
|
|
)
|
|
|
|
def deferred_resources(payload: dict[str, Any]) -> dict[str, Any]:
|
|
return next(
|
|
cell
|
|
for cell in payload["protocol"]["capabilities"]
|
|
if cell["kind"] == "capability"
|
|
and cell["role"] == "server"
|
|
and cell["surface"] == "resources"
|
|
)
|
|
|
|
# Positive control: the unmutated reviewed candidate stays projectable for both eras.
|
|
control = self.candidate()
|
|
for revision in (MODERN, LEGACY):
|
|
with self.subTest(control="unmutated-candidate", revision=revision):
|
|
completed, frozen, provenance = self.run_producer(control, revision, tag="control")
|
|
self.assertEqual(
|
|
completed.returncode, 0, completed.stdout + completed.stderr
|
|
)
|
|
self.assertTrue(frozen.is_file(), "the reviewed candidate stays projectable")
|
|
self.assertTrue(provenance.is_file(), "the reviewed candidate stays projectable")
|
|
|
|
rows = (
|
|
("client-subfeatures-null", client_sampling, {"subfeatures": None}),
|
|
("client-subfeatures-object-member", client_sampling, {"subfeatures": [{}]}),
|
|
("client-open-subfeatures-string", client_sampling, {"open_subfeatures": "false"}),
|
|
("deferred-subfeatures-null", deferred_resources, {"subfeatures": None}),
|
|
("deferred-open-subfeatures-string", deferred_resources, {"open_subfeatures": "false"}),
|
|
)
|
|
for label, select, fields in rows:
|
|
payload = self.candidate()
|
|
select(payload).update(fields)
|
|
with self.subTest(mutation=label):
|
|
self.assert_rejected_in_both_modes(payload, label, expect_named=tuple(fields))
|
|
for revision in (MODERN, LEGACY):
|
|
with self.subTest(mutation=label, revision=revision):
|
|
completed, frozen, provenance = self.run_producer(payload, revision, tag=label)
|
|
combined = completed.stdout + completed.stderr
|
|
self.assertNotIn("Traceback (most recent call last)", combined, combined)
|
|
self.assertEqual(completed.returncode, 1, combined)
|
|
self.assertIn("PROJECTION_REJECTED", combined, combined)
|
|
self.assertFalse(
|
|
frozen.exists(), "a rejected projection writes no frozen artifact"
|
|
)
|
|
self.assertFalse(
|
|
provenance.exists(), "a rejected projection writes no provenance"
|
|
)
|
|
|
|
# The same wrong type declared only in the modern override: the era that resolves to the
|
|
# malformed record is rejected, the other era keeps its valid effective record.
|
|
per_era = self.candidate()
|
|
overrides = client_sampling(per_era).setdefault("per_revision", {})
|
|
overrides[MODERN] = {**overrides.get(MODERN, {}), "subfeatures": None}
|
|
self.assert_rejected_in_both_modes(
|
|
per_era, "client-effective-subfeatures-null", expect_named=("subfeatures",)
|
|
)
|
|
completed, frozen, provenance = self.run_producer(per_era, MODERN, tag="per-era")
|
|
combined = completed.stdout + completed.stderr
|
|
self.assertNotIn("Traceback (most recent call last)", combined, combined)
|
|
self.assertEqual(completed.returncode, 1, combined)
|
|
self.assertIn("PROJECTION_REJECTED", combined, combined)
|
|
self.assertFalse(frozen.exists(), "a rejected projection writes no frozen artifact")
|
|
self.assertFalse(provenance.exists(), "a rejected projection writes no provenance")
|
|
completed, frozen, provenance = self.run_producer(per_era, LEGACY, tag="per-era")
|
|
self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr)
|
|
self.assertTrue(frozen.is_file(), "the unaffected era stays projectable")
|
|
self.assertTrue(provenance.is_file(), "the unaffected era stays projectable")
|
|
|
|
# The reviewed open maps stay projectable: the type guard must not forbid a legal declaration.
|
|
open_map = self.candidate()
|
|
next(
|
|
cell
|
|
for cell in open_map["protocol"]["capabilities"]
|
|
if cell["kind"] == "capability" and cell["role"] == "server" and cell["surface"] == "experimental"
|
|
).update(status="supported", advertised={"experimental": {"vendor.foo": {"x": True}}})
|
|
for revision in (MODERN, LEGACY):
|
|
with self.subTest(control="reviewed-open-map", revision=revision):
|
|
completed, frozen, provenance = self.run_producer(open_map, revision, tag="open-map")
|
|
self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr)
|
|
self.assertTrue(frozen.is_file(), "a reviewed open map stays projectable")
|
|
self.assertTrue(provenance.is_file(), "a reviewed open map stays projectable")
|
|
|
|
|
|
class Story31FoundationSourceContractTests(unittest.TestCase):
|
|
"""R5/R6/R8/R9: explicit GAP markers with executable source-level contract checks.
|
|
|
|
The registry dispatcher/observer/call seams do not exist on this revision: their tests are
|
|
compile-dependent, so they are **not** declared as behavioural RED. Instead each group pins the
|
|
presence of the required seam as an executable source contract; it flips to GREEN when BUILD
|
|
lands the seam, and never fakes a RED or breaks the accepted build.
|
|
"""
|
|
|
|
def registry_production_source(self) -> str:
|
|
return self.production_source_of(REGISTRY_RS)
|
|
|
|
@staticmethod
|
|
def production_source_of(path: Path) -> str:
|
|
"""The production (non-`#[cfg(test)] mod tests`) source of one crate file.
|
|
|
|
The crate declares more than one `#[cfg(test)]` item (e.g. a test-only enum variant), so the
|
|
split anchors on the trailing test module, not on the first `#[cfg(test)]` occurrence.
|
|
"""
|
|
text = path.read_text(encoding="utf-8")
|
|
parts = re.split(r"#\[cfg\(test\)\]\s*\nmod tests\b", text, maxsplit=1)
|
|
return parts[0]
|
|
|
|
def test_gap_r5_registration_is_adopted_by_the_real_dispatch(self) -> None:
|
|
"""GAP(R5): the owning registration must drive the real router, not only its own tests.
|
|
|
|
Current state: `dispatchable_for` is referenced only from the registry's own `#[cfg(test)]`
|
|
module; `src/app.rs` routes `tools/*`, `ping`, `server/discover`, `initialize` through
|
|
separate literal `match` arms, so a registered capability cannot change what is dispatched.
|
|
|
|
Exact future test + oracle (BUILD): an inert private method descriptor added to the registry
|
|
must change the real dispatch/advertisement, and an unregistered method must have no
|
|
side effect; selector `cargo test -p crank-community-mcp --lib protocol::registry::` plus the
|
|
endpoint dispatch oracle in `apps/mcp-server/tests/integration/era_isolation.rs`.
|
|
"""
|
|
adoption_tokens = ("dispatchable_for", "CapabilityRegistry", "METHOD_DESCRIPTORS")
|
|
adopted: list[str] = []
|
|
for path in sorted(MCP_CRATE_SRC.rglob("*.rs")):
|
|
relative = path.relative_to(MCP_CRATE_SRC).as_posix()
|
|
if relative in {"protocol/registry.rs", "app/tests.rs"}:
|
|
continue
|
|
text = path.read_text(encoding="utf-8")
|
|
if any(token in text for token in adoption_tokens):
|
|
adopted.append(relative)
|
|
self.assertTrue(
|
|
adopted,
|
|
"GAP(R5): no production callsite consults the owning registration, so registration cannot "
|
|
"change the real dispatch; expected a production reference to "
|
|
f"{adoption_tokens} outside protocol/registry.rs and app/tests.rs",
|
|
)
|
|
|
|
def test_gap_r6_advertisement_is_indexed_by_the_requested_era(self) -> None:
|
|
"""GAP(R6): advertisement must be computed for the requested revision.
|
|
|
|
Current state: `advertised_capabilities()` merges every descriptor eligible in the revisions
|
|
it declares, with no requested era; the single production entry
|
|
(`src/protocol/mod.rs` -> `registry::advertised_capabilities()`) therefore returns the same
|
|
object for the legacy `initialize` and the modern `server/discover` advertisements, and a
|
|
single-era descriptor leaks into the other era.
|
|
|
|
Exact future test + oracle (BUILD): a private inert modern-only descriptor must not appear in
|
|
the legacy advertisement and vice versa; selector
|
|
`cargo test -p crank-community-mcp --lib protocol::registry::`.
|
|
"""
|
|
production = self.registry_production_source()
|
|
signature = re.search(
|
|
r"fn\s+advertised_capabilities[a-z_]*\s*\(([^)]*)\)", production
|
|
)
|
|
if signature is None:
|
|
self.fail(
|
|
"GAP(R6): the registry must expose the advertisement delivery entry point "
|
|
"(`advertised_capabilities*`) in production source"
|
|
)
|
|
self.assertIn(
|
|
"revision",
|
|
signature.group(1),
|
|
"GAP(R6): the advertisement entry point must be indexed by the requested era, found "
|
|
f"signature ({signature.group(0)!r})",
|
|
)
|
|
|
|
def test_gap_r8_shared_tools_call_boundary_is_neutral(self) -> None:
|
|
"""GAP(R8): the neutral tools/call boundary must exist, not only the list arm.
|
|
|
|
Current state: `protocol/command.rs` declares `ToolsCommand::List` only; `tools/call` still
|
|
chains wire-level `message`/`ResponseMode`/`protocolVersion` -> `axum::Response` through
|
|
`tool_search.rs`/`app.rs`/`approval_policy.rs`.
|
|
|
|
Exact future test + oracle (BUILD, spec section 6.2): `ToolsCommand::Call`,
|
|
`ToolCallCommand`, `ToolCallOutcome` and a `ServiceContext` carrying the verified credential,
|
|
correlation and neutral continuation flag; the acceptance oracle is the unchanged endpoint
|
|
behaviour of `tools/call` (approval pending / unavailable / error / success) with adapters
|
|
owning the wire shape.
|
|
"""
|
|
production = self.production_source_of(COMMAND_RS)
|
|
missing = [
|
|
symbol
|
|
for symbol in ("ToolCallCommand", "ToolCallOutcome")
|
|
if symbol not in production
|
|
]
|
|
self.assertEqual(
|
|
missing,
|
|
[],
|
|
f"GAP(R8): the neutral tools/call boundary is missing {missing}",
|
|
)
|
|
|
|
def test_gap_r9_semantic_era_and_service_markers_exist(self) -> None:
|
|
"""GAP(R9): the four semantic `#[cfg(test)]` markers must exist (spec section 7.2).
|
|
|
|
Current state: none of `ADMISSION_ENTER`, `ERA_MODERN_ENTER`, `ERA_LEGACY_ENTER`,
|
|
`TOOLS_SERVICE_ENTER` is present in the crate, so a pre-admission reject cannot be
|
|
distinguished from a wrong-era dispatch, and a rejected request cannot be proven not to have
|
|
entered the wrong Tools handler (AT-06 / AT-11).
|
|
|
|
Exact future test + oracle (BUILD): `cargo test -p crank-community-mcp --lib
|
|
app::dispatch_observer::` with the per-case delta table of spec section 7.2
|
|
(pre-admission reject => 1 admission, 0 era handler, 0 service; adapter reject => exactly its
|
|
own era; valid call => correct era + service + upstream).
|
|
"""
|
|
sources = {path: path.read_text(encoding="utf-8") for path in MCP_CRATE_SRC.rglob("*.rs")}
|
|
missing = [
|
|
marker
|
|
for marker in ("ADMISSION_ENTER", "ERA_MODERN_ENTER", "ERA_LEGACY_ENTER", "TOOLS_SERVICE_ENTER")
|
|
if not any(marker in text for text in sources.values())
|
|
]
|
|
self.assertEqual(missing, [], f"GAP(R9): semantic dispatch markers are absent: {missing}")
|
|
|
|
|
|
class Story31FoundationProjectionBoundaryTests(unittest.TestCase):
|
|
"""R10: the canonical inventory -> frozen bytes -> provenance sidecar chain must be executable.
|
|
|
|
The reproducible canonical -> projection -> frozen equality itself is exercised by
|
|
`test_project_advertised_capabilities.py` (the producer runs on `docs/capability-inventory.json`);
|
|
this class pins the *committed* provenance linkage that the producer cannot fix by itself.
|
|
"""
|
|
|
|
EXPECTED_FROZEN = (
|
|
b'{"completions":{},"logging":{},"prompts":{"listChanged":true},"resources":{"listChanged":true,"subscribe":false},"tools":{"listChanged":true}}'
|
|
)
|
|
EXPECTED_FROZEN_SHA256 = "7ffa56029595fc84b6ee29930d65eada2b4d7cffd7d76f72a945fe795cabfe0f"
|
|
|
|
def test_control_committed_frozen_oracle_is_the_canonical_projection(self) -> None:
|
|
"""Positive control (existing GREEN, AT-13): the committed frozen bytes are canonical."""
|
|
for revision in (MODERN, LEGACY):
|
|
with self.subTest(revision=revision):
|
|
payload = (
|
|
MCP_REVISION_FIXTURES / f"{revision}-advertised-capabilities.frozen.json"
|
|
).read_bytes()
|
|
self.assertEqual(payload, self.EXPECTED_FROZEN)
|
|
self.assertEqual(hashlib.sha256(payload).hexdigest(), self.EXPECTED_FROZEN_SHA256)
|
|
|
|
def test_r10_committed_provenance_sidecar_binds_the_canonical_inventory(self) -> None:
|
|
"""The committed per-era sidecar must record the digest of the canonical inventory.
|
|
|
|
Current state: both frozen files are produced correctly from the canonical inventory, but the
|
|
committed sidecars record the digest of the reviewed *candidate* fixture
|
|
(`inventory-reviewed-v2-candidate.json`), not of `docs/capability-inventory.json`, so the
|
|
provenance chain is not executable from the canonical artifact.
|
|
"""
|
|
canonical_digest = hashlib.sha256(CANONICAL_INVENTORY.read_bytes()).hexdigest()
|
|
candidate_digest = hashlib.sha256(CANDIDATE.read_bytes()).hexdigest()
|
|
self.assertNotEqual(
|
|
canonical_digest,
|
|
candidate_digest,
|
|
"the canonical inventory and the reviewed candidate must stay distinct artifacts",
|
|
)
|
|
for revision in (MODERN, LEGACY):
|
|
with self.subTest(revision=revision):
|
|
sidecar = json.loads(
|
|
(
|
|
MCP_REVISION_FIXTURES / f"{revision}-advertised-capabilities.provenance.json"
|
|
).read_text(encoding="utf-8")
|
|
)
|
|
self.assertEqual(
|
|
sidecar["inventory_sha256"],
|
|
canonical_digest,
|
|
"the committed sidecar must bind the canonical inventory digest "
|
|
f"({canonical_digest}), not the candidate fixture digest ({candidate_digest})",
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|