Files
crank/tests/unit/test_validate_capability_inventory.py
T
bsodfather 54ce770682
CI / Rust Checks (pull_request) Failing after 13m41s
CI / UI Checks (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Deploy (pull_request) Skipped
Community Release / release (push) Failing after 13m33s
feat(epic3): deliver the accepted Epic 3 scope through Story 3.17
Recovery delivery of the accumulated, TL-accepted Epic 3 work that had never been
committed to Git (stories 3.6-3.17, see the crank board). Story 3.18 is not accepted
yet and is deliberately excluded: its logging surface, its capability registration,
its oracle projections and its tests are not part of this tree.

Accepted scope
- 3.6 typed static Resource Drafts; 3.7 publication and exact binding of static
  Resource Versions; 3.8 MCP discovery/read and revocation of static Resources;
- 3.9 Resource import/export in API and UI plus TLS dependency maintenance;
  3.10 REST-backed Resource templates; 3.11 shared REST read runtime and the MCP
  revision/capability foundation;
- 3.12 parameterised Prompt Drafts with preview; 3.13 Prompt publication and exact
  version binding to Agents; 3.14 MCP prompts; 3.15 Prompt import/export;
- 3.16 composite Prompts with published Resources (dependency content scan plus
  dynamic read through the accepted 3.11 runtime, admin preview and MCP prompts/get);
- 3.17 bounded MCP completion suggestions from typed published schemas and the
  resources/list admission fix for bound template publications.

Accepted evidence being reused (unchanged code paths)
- 3.16 candidate: core 7, admin-api 22, registry 4, mcp-server 4, UI 6; one final
  `just verify` run 1651 passed / 0 failed / 2 ignored.
- 3.17 candidate: full run 1672 passed / 0 failed / 2 ignored, QA targeted reruns and
  a real MCP Inspector 2.12.0 client smoke.

Delivery engineering in this commit
- CI: one event per change (feature branches are gated by their pull request, push
  runs only on main), story events run the changed packages, the full workspace suite
  stays the feature/release gate; no cache/sharding without a measured runner check.
- Release: the packaged archives, checksums and SBOM are now uploaded to the Gitea
  release entry of the tag instead of dying with the runner workspace.

Not included on purpose: Story 3.18 work, build caches, runtime databases, local
credentials and scratch evidence.

Refs: crank kanban t_e39d7263; baseline 1b6db09bd1.
2026-10-25 09:08:22 +00:00

1587 lines
74 KiB
Python

import copy
import importlib.util
import json
import re
import shutil
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[2]
VALIDATOR = ROOT / "scripts" / "validate-capability-inventory.py"
CANONICAL_INVENTORY = ROOT / "docs" / "capability-inventory.json"
CANONICAL_SCHEMA = ROOT / "docs" / "schemas" / "capability-inventory.schema.json"
COMMUNITY_CHECKER = ROOT / "scripts" / "check-community-scope.py"
COMMUNITY_README = ROOT / "README.md"
DOCS_INDEX = ROOT / "docs" / "README.md"
COMMUNITY_INTRO = ROOT / "docs" / "intro.md"
COMMUNITY_README_EN = ROOT / "docs" / "en" / "README.md"
JUSTFILE = ROOT / "justfile"
CI_WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
RELEASE_WORKFLOW = ROOT / ".gitea" / "workflows" / "release.yml"
REQUIRED_SURFACES = ROOT / "docs" / "capability-baseline" / "required-surfaces.json"
MCP_REVISION_FIXTURES = ROOT / "tests" / "fixtures" / "mcp-revisions"
def load_validator_module():
spec = importlib.util.spec_from_file_location("capability_inventory_validator", VALIDATOR)
if spec is None or spec.loader is None:
raise RuntimeError("cannot load capability inventory validator")
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
class CapabilityInventoryValidatorTests(unittest.TestCase):
def setUp(self) -> None:
self._temporary_directory = tempfile.TemporaryDirectory()
self.root = Path(self._temporary_directory.name)
self.inventory_path = self.root / "docs" / "capability-inventory.json"
self.schema_path = (
self.root / "docs" / "schemas" / "capability-inventory.schema.json"
)
self.inventory_path.parent.mkdir(parents=True)
self.schema_path.parent.mkdir(parents=True)
def tearDown(self) -> None:
self._temporary_directory.cleanup()
def write_evidence(self, relative_path: str, content: str = "verified\n") -> str:
path = self.root / relative_path
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
return relative_path
def valid_flow(
self,
flow_id: str,
flow_type: str,
requirement: str,
status: str = "implemented",
capabilities: list[str] | None = None,
) -> dict[str, object]:
automated = self.write_evidence(
f"evidence/automated/{flow_id}.txt",
f"{flow_id}: automated acceptance evidence\n",
)
manual = self.write_evidence(
f"evidence/manual/{flow_id}.md",
f"# {flow_id}\n\nManual verification complete.\n",
)
return {
"id": flow_id,
"type": flow_type,
"requirements": [requirement],
"user_outcome": f"User completes {flow_id.replace('-', ' ')}.",
"owner": "community-release",
"status": status,
"capabilities": capabilities or ["tools"],
"evidence": {
"automated": [automated],
"manual": [manual],
},
}
def write_inventory(self, flows: list[dict[str, object]]) -> None:
payload = {
"schema_version": 1,
"product": "crank-community",
"flows": flows,
}
self.inventory_path.write_text(
json.dumps(payload, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
def install_canonical_schema(self) -> None:
self.assertTrue(
CANONICAL_SCHEMA.is_file(),
f"Story 1.1 must create {CANONICAL_SCHEMA.relative_to(ROOT)}",
)
shutil.copyfile(CANONICAL_SCHEMA, self.schema_path)
def run_validator(
self,
required_frs: list[str],
install_schema: bool = True,
mode: str | None = None,
) -> subprocess.CompletedProcess[str]:
if install_schema:
self.install_canonical_schema()
command = [
sys.executable,
str(VALIDATOR),
"--root",
str(self.root),
"--inventory",
str(self.inventory_path.relative_to(self.root)),
"--schema",
str(self.schema_path.relative_to(self.root)),
]
for requirement in required_frs:
command.extend(["--required-fr", requirement])
if mode is not None:
command.extend(["--mode", mode])
return subprocess.run(
command,
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
def test_accepts_complete_inventory_with_all_flow_types_and_statuses(self) -> None:
flows = [
self.valid_flow("ui-create-method", "ui", "FR-1", "implemented"),
self.valid_flow("api-create-method", "api", "FR-2", "planned"),
self.valid_flow("mcp-tool-call", "mcp", "FR-3", "gap"),
self.valid_flow("mcp-resource-read", "mcp", "FR-4", "blocked"),
]
self.write_inventory(flows)
result = self.run_validator(["FR-1", "FR-2", "FR-3", "FR-4"])
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn("Capability inventory validation passed", result.stdout)
self.assertRegex(result.stdout, r"\btotal=4\b")
self.assertRegex(result.stdout, r"\bimplemented=1\b")
self.assertRegex(result.stdout, r"\bplanned=1\b")
self.assertRegex(result.stdout, r"\bgap=1\b")
self.assertRegex(result.stdout, r"\bblocked=1\b")
def test_counts_only_implemented_flows_as_pass(self) -> None:
flows = [
self.valid_flow("implemented-flow", "api", "FR-1", "implemented"),
self.valid_flow("planned-flow", "ui", "FR-2", "planned"),
self.valid_flow("gap-flow", "mcp", "FR-3", "gap"),
self.valid_flow("blocked-flow", "mcp", "FR-4", "blocked"),
]
self.write_inventory(flows)
result = self.run_validator(["FR-1", "FR-2", "FR-3", "FR-4"])
self.assertEqual(result.returncode, 0, result.stderr)
self.assertRegex(result.stdout, r"\bpass=1\b")
self.assertNotRegex(result.stdout, r"\bpass=4\b")
def test_rejects_duplicate_id_unknown_enum_and_missing_required_fields(self) -> None:
base = self.valid_flow("flow-a", "api", "FR-1")
cases: list[tuple[str, list[dict[str, object]], str]] = []
duplicate = copy.deepcopy(base)
duplicate["user_outcome"] = "A distinct outcome with a duplicate identifier."
cases.append(("duplicate ID", [base, duplicate], "DUPLICATE_FLOW_ID"))
unknown_status = copy.deepcopy(base)
unknown_status["status"] = "done"
cases.append(("unknown status", [unknown_status], "UNKNOWN_STATUS"))
unknown_type = copy.deepcopy(base)
unknown_type["type"] = "worker"
cases.append(("unknown type", [unknown_type], "UNKNOWN_FLOW_TYPE"))
no_owner = copy.deepcopy(base)
no_owner.pop("owner")
cases.append(("missing owner", [no_owner], "MISSING_OWNER"))
no_outcome = copy.deepcopy(base)
no_outcome.pop("user_outcome")
cases.append(("missing required field", [no_outcome], "MISSING_REQUIRED_FIELD"))
for label, flows, expected_code in cases:
with self.subTest(label=label):
self.write_inventory(flows)
result = self.run_validator(["FR-1"])
self.assertNotEqual(result.returncode, 0)
self.assertIn(f"error[{expected_code}]", result.stderr)
def test_rejects_missing_mandatory_requirement(self) -> None:
self.write_inventory([self.valid_flow("only-fr-one", "api", "FR-1")])
result = self.run_validator(["FR-1", "FR-46"])
self.assertNotEqual(result.returncode, 0)
self.assertIn("error[MISSING_REQUIRED_FR]", result.stderr)
self.assertIn("FR-46", result.stderr)
def test_rejects_missing_non_regular_and_escaping_evidence_links(self) -> None:
base = self.valid_flow("evidence-flow", "mcp", "FR-46")
outside = self.root.parent / f"{self.root.name}-outside.txt"
outside.write_text("outside\n", encoding="utf-8")
self.addCleanup(outside.unlink, missing_ok=True)
internal_target = self.write_evidence("evidence/automated/internal-target.txt")
internal_link = self.root / "evidence" / "automated" / "internal-link.txt"
internal_link.symlink_to(self.root / internal_target)
outside_link = self.root / "evidence" / "automated" / "outside-link.txt"
outside_link.symlink_to(outside)
broken_cases = [
"evidence/automated/does-not-exist.txt",
"evidence/automated",
f"../{outside.name}",
str(outside),
"evidence/automated/internal-link.txt",
"evidence/automated/outside-link.txt",
]
for evidence_path in broken_cases:
with self.subTest(evidence_path=evidence_path):
flow = copy.deepcopy(base)
flow["evidence"] = {
"automated": [evidence_path],
"manual": ["evidence/manual/evidence-flow.md"],
}
self.write_inventory([flow])
result = self.run_validator(["FR-46"])
self.assertNotEqual(result.returncode, 0)
self.assertIn("error[BROKEN_EVIDENCE_LINK]", result.stderr)
def test_report_is_deterministic_bounded_sorted_and_redacted(self) -> None:
secret_canary = "SECRET_CANARY_sk-live-4hQ9Vw8x"
flows = []
for index in range(1_200, 0, -1):
flow = self.valid_flow(
f"invalid-flow-{index:04d}",
"api",
f"FR-{index}",
)
flow["status"] = "unknown"
flow["user_outcome"] = f"{secret_canary}:{index}"
flows.append(flow)
self.write_inventory(flows)
raw_inventory = self.inventory_path.read_text(encoding="utf-8")
first = self.run_validator(["FR-46"])
second = self.run_validator(["FR-46"])
self.assertNotEqual(first.returncode, 0)
self.assertEqual(first.stderr, second.stderr)
self.assertLessEqual(len(first.stderr.encode("utf-8")), 64 * 1024)
self.assertNotIn(secret_canary, first.stderr)
self.assertNotIn(raw_inventory, first.stderr)
diagnostics = re.findall(r"error\[([^]]+)\] ([^:]+):", first.stderr)
self.assertTrue(diagnostics)
self.assertEqual(diagnostics[:-1], sorted(diagnostics[:-1]))
self.assertEqual(diagnostics[-1][0], "TRUNCATED")
self.assertNotRegex(first.stderr, r"invalid-flow-\d{4}")
def test_rejects_malformed_documents_and_schema_contract_drift(self) -> None:
self.write_inventory([self.valid_flow("valid-flow", "api", "FR-46")])
self.inventory_path.write_text("{not-json", encoding="utf-8")
malformed_inventory = self.run_validator(["FR-46"])
self.assertNotEqual(malformed_inventory.returncode, 0)
self.assertIn("error[INVALID_JSON]", malformed_inventory.stderr)
self.write_inventory([self.valid_flow("valid-flow", "api", "FR-46")])
self.schema_path.write_text("{not-json", encoding="utf-8")
malformed_schema = self.run_validator(["FR-46"], install_schema=False)
self.assertNotEqual(malformed_schema.returncode, 0)
self.assertIn("error[INVALID_SCHEMA]", malformed_schema.stderr)
self.install_canonical_schema()
schema = json.loads(self.schema_path.read_text(encoding="utf-8"))
schema["$schema"] = "https://json-schema.org/draft/2019-09/schema"
self.schema_path.write_text(json.dumps(schema), encoding="utf-8")
drifted_schema = self.run_validator(["FR-46"], install_schema=False)
self.assertNotEqual(drifted_schema.returncode, 0)
self.assertIn("error[INVALID_SCHEMA_CONTRACT]", drifted_schema.stderr)
def test_rejects_duplicate_nonstandard_recursive_and_large_integer_json(self) -> None:
malformed_documents = {
"duplicate member": (
'{"schema_version":1,"schema_version":1,'
'"product":"crank-community","flows":[]}',
"INVALID_JSON",
),
"non-standard number": (
'{"schema_version":1,"product":NaN,"flows":[]}',
"INVALID_JSON",
),
"recursive": ("[" * 100_000 + "0" + "]" * 100_000, "INVALID_JSON"),
"large integer": ("9" * 5_000, "INVALID_JSON"),
}
for label, (content, expected_code) in malformed_documents.items():
with self.subTest(label=label):
self.inventory_path.write_text(content, encoding="utf-8")
result = self.run_validator(["FR-46"])
self.assertNotEqual(result.returncode, 0)
self.assertIn(f"error[{expected_code}]", result.stderr)
self.assertNotIn("Traceback", result.stderr)
self.assertLessEqual(len(result.stderr.encode("utf-8")), 64 * 1024)
def test_schema_parity_rejects_required_type_ref_and_pattern_drift(self) -> None:
self.write_inventory([self.valid_flow("schema-parity", "api", "FR-46")])
mutations = {
"root required missing": lambda schema: schema.pop("required"),
"root contract inversion": lambda schema: schema.update({"not": {}}),
"malformed evidence required": lambda schema: schema["$defs"]["evidence"].update(
{"required": [{}]}
),
"flow ref drift": lambda schema: schema["properties"]["flows"]["items"].update(
{"$ref": "#/$defs/evidence"}
),
"owner type drift": lambda schema: schema["$defs"]["flow"]["properties"][
"owner"
].update({"type": "integer"}),
"owner const drift": lambda schema: schema["$defs"]["flow"]["properties"][
"owner"
].update({"const": "impossible"}),
"nested applicator drift": lambda schema: schema["$defs"]["evidencePaths"][
"items"
].update({"anyOf": [{"type": "integer"}]}),
"evidence pattern drift": lambda schema: schema["$defs"]["evidencePaths"][
"items"
].update({"pattern": ".+"}),
}
for label, mutate in mutations.items():
with self.subTest(label=label):
self.install_canonical_schema()
schema = json.loads(self.schema_path.read_text(encoding="utf-8"))
mutate(schema)
self.schema_path.write_text(json.dumps(schema), encoding="utf-8")
result = self.run_validator(["FR-46"], install_schema=False)
self.assertNotEqual(result.returncode, 0)
self.assertIn("error[INVALID_SCHEMA_CONTRACT]", result.stderr)
self.assertNotIn("Traceback", result.stderr)
self.install_canonical_schema()
schema = json.loads(self.schema_path.read_text(encoding="utf-8"))
schema["$defs"]["flow"]["properties"]["status"]["enum"].reverse()
self.schema_path.write_text(json.dumps(schema), encoding="utf-8")
reordered_enum = self.run_validator(["FR-46"], install_schema=False)
self.assertEqual(reordered_enum.returncode, 0, reordered_enum.stderr)
def test_rejects_boolean_schema_version_and_whitespace_required_text(self) -> None:
base = self.valid_flow("strict-scalars", "api", "FR-46")
payload_cases: list[tuple[str, dict[str, object], str]] = []
boolean_version = {
"schema_version": True,
"product": "crank-community",
"flows": [copy.deepcopy(base)],
}
payload_cases.append(("boolean inventory version", boolean_version, "MISSING_REQUIRED_FIELD"))
whitespace_owner = copy.deepcopy(base)
whitespace_owner["owner"] = " \t\n "
payload_cases.append(
(
"whitespace owner",
{"schema_version": 1, "product": "crank-community", "flows": [whitespace_owner]},
"MISSING_OWNER",
)
)
whitespace_outcome = copy.deepcopy(base)
whitespace_outcome["user_outcome"] = " \t\n "
payload_cases.append(
(
"whitespace outcome",
{"schema_version": 1, "product": "crank-community", "flows": [whitespace_outcome]},
"LIMIT_EXCEEDED",
)
)
for label, payload, expected_code in payload_cases:
with self.subTest(label=label):
self.inventory_path.write_text(json.dumps(payload), encoding="utf-8")
result = self.run_validator(["FR-46"])
self.assertNotEqual(result.returncode, 0)
self.assertIn(f"error[{expected_code}]", result.stderr)
self.write_inventory([base])
self.install_canonical_schema()
schema = json.loads(self.schema_path.read_text(encoding="utf-8"))
schema["properties"]["schema_version"]["const"] = True
self.schema_path.write_text(json.dumps(schema), encoding="utf-8")
schema_result = self.run_validator(["FR-46"], install_schema=False)
self.assertNotEqual(schema_result.returncode, 0)
self.assertIn("error[INVALID_SCHEMA_CONTRACT]", schema_result.stderr)
def test_diagnostic_accumulation_is_capped_counted_and_redacted(self) -> None:
secret_canary = "SECRET_CANARY_unknown-field"
flow = self.valid_flow("bounded-diagnostics", "api", "FR-46")
for index in range(2_000):
flow[f"{secret_canary}-{index:04d}"] = True
self.write_inventory([flow])
result = self.run_validator(["FR-46"])
self.assertNotEqual(result.returncode, 0)
self.assertIn("error[TRUNCATED] /: diagnostics omitted=1000", result.stderr)
self.assertEqual(result.stderr.count("error[UNKNOWN_FIELD]"), 1_000)
self.assertNotIn(secret_canary, result.stderr)
self.assertLessEqual(len(result.stderr.encode("utf-8")), 64 * 1024)
def test_rejects_oversized_document_and_field_limits(self) -> None:
self.inventory_path.write_bytes(b" " * (4_194_304 + 1))
oversized = self.run_validator(["FR-46"])
self.assertNotEqual(oversized.returncode, 0)
self.assertIn("error[INPUT_TOO_LARGE]", oversized.stderr)
flow = self.valid_flow("bounded-flow", "api", "FR-46")
flow["user_outcome"] = "x" * 4_097
self.write_inventory([flow])
overlong = self.run_validator(["FR-46"])
self.assertNotEqual(overlong.returncode, 0)
self.assertIn("error[LIMIT_EXCEEDED]", overlong.stderr)
def test_named_string_limits_cover_below_at_and_above_boundaries(self) -> None:
string_cases = [
("id", 128, lambda length: "a" * length),
("owner", 256, lambda length: "o" * length),
("user_outcome", 4_096, lambda length: "u" * length),
("notes", 4_096, lambda length: "n" * length),
("requirements", 128, lambda length: "FR-" + "1" * (length - 3)),
]
for field, limit, make_value in string_cases:
for delta in (-1, 0, 1):
with self.subTest(field=field, length=limit + delta):
safe_field = field.replace("_", "-")
flow = self.valid_flow(f"bounded-{safe_field}-{delta + 1}", "api", "FR-46")
value = make_value(limit + delta)
if field == "requirements":
flow[field] = [value]
required_frs = [value]
else:
flow[field] = value
required_frs = ["FR-46"]
self.write_inventory([flow])
result = self.run_validator(required_frs)
if delta <= 0:
self.assertEqual(result.returncode, 0, result.stderr)
else:
self.assertNotEqual(result.returncode, 0)
self.assertIn("error[LIMIT_EXCEEDED]", result.stderr)
def test_collection_limits_cover_below_at_and_above_boundaries(self) -> None:
module = load_validator_module()
automated = self.write_evidence("evidence/shared/automated.txt")
manual = self.write_evidence("evidence/shared/manual.md")
def direct_flow(index: int) -> dict[str, object]:
return {
"id": f"flow-{index:05d}",
"type": "api",
"requirements": ["FR-46"],
"user_outcome": "A bounded flow succeeds.",
"owner": "community-release",
"status": "implemented",
"capabilities": ["tools"],
"evidence": {"automated": [automated], "manual": [manual]},
}
all_flows = [direct_flow(index) for index in range(10_001)]
for count in (9_999, 10_000, 10_001):
with self.subTest(collection="flows", count=count):
inventory = {
"schema_version": 1,
"product": "crank-community",
"flows": all_flows[:count],
}
diagnostics, _ = module.validate_inventory(self.root, inventory, ["FR-46"])
has_limit_error = any(
item.code == "LIMIT_EXCEEDED" and item.pointer == "/flows"
for item in diagnostics.ordered()
)
self.assertEqual(has_limit_error, count > 10_000)
for count in (63, 64, 65):
with self.subTest(collection="requirements", count=count):
flow = self.valid_flow(f"requirements-{count}", "api", "FR-1")
flow["requirements"] = [f"FR-{index}" for index in range(1, count + 1)]
self.write_inventory([flow])
result = self.run_validator(["FR-1"])
self.assertEqual(result.returncode == 0, count <= 64, result.stderr)
evidence_paths = [
self.write_evidence(f"evidence/bounds/path-{index:02d}.txt")
for index in range(65)
]
for count in (63, 64, 65):
with self.subTest(collection="evidence", count=count):
flow = self.valid_flow(f"evidence-{count}", "api", "FR-46")
flow["evidence"] = {
"automated": evidence_paths[:count],
"manual": [manual],
}
self.write_inventory([flow])
result = self.run_validator(["FR-46"])
self.assertEqual(result.returncode == 0, count <= 64, result.stderr)
for count in (15, 16, 17):
with self.subTest(collection="capability-bound", count=count):
diagnostics = module.validate_string_list(
[f"capability-{index}" for index in range(count)],
"/capabilities",
16,
128,
)
has_limit_error = any(item.code == "LIMIT_EXCEEDED" for item in diagnostics)
self.assertEqual(has_limit_error, count > 16)
def test_evidence_path_limit_covers_below_at_and_above_boundary(self) -> None:
def path_of_length(length: int) -> str:
segment_count = (length + 199) // 201
character_count = length - (segment_count - 1)
lengths: list[int] = []
remaining = character_count
for index in range(segment_count):
slots_left = segment_count - index - 1
current = min(200, remaining - slots_left)
lengths.append(current)
remaining -= current
path = "/".join("p" * size for size in lengths)
self.assertEqual(len(path), length)
return path
manual = self.write_evidence("evidence/manual/path-limit.md")
for length in (1_023, 1_024, 1_025):
with self.subTest(length=length):
evidence_path = path_of_length(length)
if length <= 1_024:
self.write_evidence(evidence_path)
flow = self.valid_flow(f"path-limit-{length}", "api", "FR-46")
flow["evidence"] = {
"automated": [evidence_path],
"manual": [manual],
}
self.write_inventory([flow])
result = self.run_validator(["FR-46"])
self.assertEqual(result.returncode == 0, length <= 1_024, result.stderr)
def test_semantic_validator_enforces_schema_shape_uniqueness_and_fr_format(self) -> None:
cases: list[tuple[str, object, str]] = []
unknown_top = {
"schema_version": 1,
"product": "crank-community",
"flows": [self.valid_flow("unknown-top", "api", "FR-46")],
"unexpected": True,
}
cases.append(("unknown top-level field", unknown_top, "UNKNOWN_FIELD"))
unknown_flow = self.valid_flow("unknown-flow", "api", "FR-46")
unknown_flow["unexpected"] = True
cases.append(
(
"unknown flow field",
{"schema_version": 1, "product": "crank-community", "flows": [unknown_flow]},
"UNKNOWN_FIELD",
)
)
duplicate_requirement = self.valid_flow("duplicate-fr", "api", "FR-46")
duplicate_requirement["requirements"] = ["FR-46", "FR-46"]
cases.append(
(
"duplicate requirement",
{
"schema_version": 1,
"product": "crank-community",
"flows": [duplicate_requirement],
},
"DUPLICATE_LIST_ITEM",
)
)
malformed_requirement = self.valid_flow("malformed-fr", "api", "FR-46")
malformed_requirement["requirements"] = ["FR-0"]
cases.append(
(
"malformed requirement",
{
"schema_version": 1,
"product": "crank-community",
"flows": [malformed_requirement],
},
"INVALID_FORMAT",
)
)
unknown_evidence = self.valid_flow("unknown-evidence", "api", "FR-46")
evidence = unknown_evidence["evidence"]
self.assertIsInstance(evidence, dict)
evidence["unexpected"] = []
cases.append(
(
"unknown evidence field",
{
"schema_version": 1,
"product": "crank-community",
"flows": [unknown_evidence],
},
"UNKNOWN_FIELD",
)
)
for label, payload, expected_code in cases:
with self.subTest(label=label):
self.inventory_path.write_text(
json.dumps(payload, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
result = self.run_validator(["FR-46"])
self.assertNotEqual(result.returncode, 0)
self.assertIn(f"error[{expected_code}]", result.stderr)
def test_accepts_planned_modern_mcp_and_load_run_capabilities(self) -> None:
capabilities = ["resources", "prompts", "tasks", "load_runs"]
flows = [
self.valid_flow(
f"planned-{capability.replace('_', '-')}",
"mcp" if capability != "load_runs" else "ui",
f"FR-{index}",
"planned",
[capability],
)
for index, capability in enumerate(capabilities, start=47)
]
self.write_inventory(flows)
result = self.run_validator([f"FR-{index}" for index in range(47, 51)])
self.assertEqual(result.returncode, 0, result.stderr)
for capability in capabilities:
self.assertNotIn(f"error[FORBIDDEN_COMMUNITY_CAPABILITY]: {capability}", result.stderr)
def test_rejects_forbidden_community_capabilities_despite_allow_text(self) -> None:
forbidden = [
"multi_workspace", # community-scope: allow=multi-workspace
"enterprise_rbac", # community-scope: allow=enterprise
"sso", # community-scope: allow=sso
"non_rest_upstream", # community-scope: allow=non-rest-upstream
"arbitrary_distributed_load_targets", # community-scope: allow=distributed-load-targets
]
for index, capability in enumerate(forbidden, start=1):
with self.subTest(capability=capability):
flow = self.valid_flow(
f"forbidden-{index}",
"api",
"FR-46",
"planned",
[capability],
)
flow["notes"] = f"community-scope: allow={capability}"
self.write_inventory([flow])
result = self.run_validator(["FR-46"])
self.assertNotEqual(result.returncode, 0)
self.assertIn("error[FORBIDDEN_COMMUNITY_CAPABILITY]", result.stderr)
self.assertIn(capability, result.stderr)
def test_scope_scanner_receives_inventory_files_explicitly_and_cannot_be_bypassed(self) -> None:
clean_inventory = self.valid_flow(
"planned-resources",
"mcp",
"FR-47",
"planned",
["resources"],
)
self.write_inventory([clean_inventory])
explicit_files = [
str(self.inventory_path.relative_to(self.root)),
"docs/new-untracked-scope-note.md",
]
note = self.root / explicit_files[1]
note.write_text("Community Resources remain planned.\n", encoding="utf-8")
clean = subprocess.run(
[
sys.executable,
str(COMMUNITY_CHECKER),
"--root",
str(self.root),
"--files",
*explicit_files,
],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
self.assertEqual(clean.returncode, 0, clean.stderr)
self.assertIn("2 files scanned", clean.stdout)
note.write_text(
"community-scope: allow=enterprise\n"
"enterprise_rbac remains outside Community scope.\n", # community-scope: allow=enterprise
encoding="utf-8",
)
rejected = subprocess.run(
[
sys.executable,
str(COMMUNITY_CHECKER),
"--root",
str(self.root),
"--files",
*explicit_files,
],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
self.assertNotEqual(rejected.returncode, 0)
self.assertIn("docs/new-untracked-scope-note.md:2", rejected.stderr)
self.assertIn("forbidden marker", rejected.stderr)
self.assertIn("enterprise", rejected.stderr) # community-scope: allow=enterprise
def test_canonical_inventory_seed_does_not_promote_planned_work(self) -> None:
for path in [
CANONICAL_INVENTORY,
CANONICAL_SCHEMA,
]:
self.assertTrue(path.is_file(), f"Missing canonical artifact: {path}")
inventory = json.loads(CANONICAL_INVENTORY.read_text(encoding="utf-8"))
schema = json.loads(CANONICAL_SCHEMA.read_text(encoding="utf-8"))
flow_schema = schema["$defs"]["flow"]
evidence_paths = schema["$defs"]["evidencePaths"]
self.assertEqual(
schema["$schema"],
"https://json-schema.org/draft/2020-12/schema",
)
self.assertEqual(schema["properties"]["flows"]["maxItems"], 10_000)
self.assertEqual(
set(flow_schema["properties"]["type"]["enum"]),
{"ui", "api", "mcp"},
)
self.assertEqual(
set(flow_schema["properties"]["status"]["enum"]),
{"implemented", "planned", "gap", "blocked"},
)
self.assertEqual(evidence_paths["minItems"], 1)
self.assertEqual(evidence_paths["maxItems"], 64)
target_capabilities = {"resources", "prompts", "tasks", "load_runs"}
matching_flows = [
flow
for flow in inventory["flows"]
if target_capabilities.intersection(flow.get("capabilities", []))
]
observed_capabilities = {
capability
for flow in matching_flows
for capability in flow.get("capabilities", [])
if capability in target_capabilities
}
self.assertEqual(observed_capabilities, target_capabilities)
self.assertTrue(matching_flows)
self.assertTrue(
all(flow["status"] != "implemented" for flow in matching_flows),
"Story 1.1 must not claim target capabilities are implemented.",
)
def test_canonical_inventory_covers_all_requirements_and_required_surfaces(self) -> None:
inventory = json.loads(CANONICAL_INVENTORY.read_text(encoding="utf-8"))
required_surfaces = json.loads(REQUIRED_SURFACES.read_text(encoding="utf-8"))
flows = {flow["id"]: flow for flow in inventory["flows"]}
observed_requirements = {
requirement for flow in flows.values() for requirement in flow["requirements"]
}
self.assertEqual(
{f"FR-{number}" for number in range(1, 55)} - observed_requirements,
set(),
)
self.assertEqual(
set(required_surfaces["required_flow_ids"]) - set(flows),
set(),
)
for flow_id in required_surfaces["required_flow_ids"]:
flow = flows[flow_id]
self.assertIn(flow["status"], {"implemented", "blocked"})
for evidence_kind in ("automated", "manual"):
for evidence_path in flow["evidence"][evidence_kind]:
resolved = ROOT / evidence_path
self.assertTrue(resolved.is_file(), evidence_path)
self.assertFalse(resolved.is_symlink(), evidence_path)
def test_tracked_documentation_describes_planned_scope(self) -> None:
for documentation in [
COMMUNITY_README,
DOCS_INDEX,
COMMUNITY_INTRO,
COMMUNITY_README_EN,
]:
text = documentation.read_text(encoding="utf-8")
self.assertIn("docs/capability-inventory.json", text)
self.assertRegex(text.lower(), r"\b(planned|gap|blocked)\b")
def test_no_story_acceptance_scenario_remains_skipped(self) -> None:
for name, method in vars(type(self)).items():
if not name.startswith("test_") or name == self._testMethodName:
continue
self.assertFalse(
getattr(method, "__unittest_skip__", False),
f"Story acceptance scenario remains skipped: {name}",
)
def test_canonical_validator_is_a_ci_and_release_prerequisite(self) -> None:
justfile = JUSTFILE.read_text(encoding="utf-8")
ci = CI_WORKFLOW.read_text(encoding="utf-8")
release = RELEASE_WORKFLOW.read_text(encoding="utf-8")
command = "scripts/validate-capability-inventory.py"
self.assertIn("capability-inventory-check:", justfile)
self.assertIn("just capability-inventory-check", justfile)
self.assertLess(ci.index("Run tooling unit tests"), ci.index(command))
self.assertLess(ci.index(command), ci.index("Check Community scope"))
self.assertIn("python3 --version", release)
self.assertLess(release.index("Run tooling unit tests"), release.index(command))
self.assertLess(release.index(command), release.index("Check Community scope"))
self.assertLess(release.index("Check Community scope"), release.index("Build release binaries"))
# --- Story 3.1 ATDD (red): the approved two-revision MCP window -------------------------------
# Oracle: tests/fixtures/mcp-revisions/* (owner-approved 2026-10-11 contract, source-backed, not a
# snapshot of current output). Behaviour is observed on the real validator
# (scripts/validate-capability-inventory.py) and the real schema copy.
# The protocol vocabulary here (supported/not_applicable/experimental/deferred) is deliberately
# separate from the unchanged flow vocabulary (implemented/planned/gap/blocked).
PROTOCOL_STATUSES = {"supported", "not_applicable", "experimental", "deferred"}
def load_revision_fixture(self, name: str) -> dict[str, object]:
return json.loads((MCP_REVISION_FIXTURES / name).read_text(encoding="utf-8"))
def write_revision_inventory(self, name: str) -> tuple[dict[str, object], dict[str, object]]:
payload = self.load_revision_fixture(name)
expected = payload.pop("expected", {})
for flow in payload.get("flows", []): # type: ignore[union-attr]
for kind in ("automated", "manual"):
for relative in flow.get("evidence", {}).get(kind, []):
self.write_evidence(relative)
self.inventory_path.write_text(
json.dumps(payload, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return payload, expected # type: ignore[return-value]
def required_surfaces(self) -> set[str]:
oracle = self.load_revision_fixture("source-backed-capabilities.json")
return {
item["surface"]
for item in oracle["items"] # type: ignore[index]
if item["role"] == "server"
}
def test_at_3_1_01_full_reviewed_candidate_passes_the_validity_gate(self) -> None:
"""Target oracle: the FULL reviewed candidate is an accepted v2 inventory (Gate A, exit 0).
This is a genuine behavioural RED on the current revision: the validator has no v2 `protocol`
block, so it answers `schema_version 1 is required` / `UNKNOWN_FIELD`. The assertions below
describe the agreed target behaviour and are deliberately NOT relaxed to the current one.
"""
self.write_candidate_inventory()
result = self.run_validator(["FR-13"])
combined = result.stdout + result.stderr
self.assertEqual(
result.returncode,
0,
"the complete reviewed v2 candidate must pass the validity gate, observed: " + combined,
)
# The agreed validity output is the existing one-line summary plus the protocol counts.
self.assertIn("Capability inventory validation passed", combined)
self.assertRegex(combined, r"\bprotocol_items=50\b")
self.assertRegex(combined, r"\brevisions=2\b")
def test_at_3_1_01_scoped_pre_atdd_candidate_stays_a_scoped_input(self) -> None:
"""Preserved pre-ATDD input: the scoped window candidate is NOT promoted to full validity.
Pure oracle-input provenance - no validator run, so this is not labelled validator GREEN. The
file keeps its own recorded verdict (`counts_as_exhaustive: false`, completeness BLOCKED) and
stays byte-identical (checked by the ATDD snapshot verifier); the reviewed candidate above is
the completeness basis.
"""
payload, expected = self.write_revision_inventory("inventory-two-revision-window.json")
protocol: Any = payload["protocol"]
cells = protocol["capabilities"]
self.assertEqual(payload["schema_version"], 2)
self.assertEqual(
[entry["revision"] for entry in protocol["revisions"]],
["2026-07-28", "2025-11-25"],
)
self.assertEqual(
set(entry["revision"] for entry in protocol["revisions"])
& set(protocol["withdrawn"]),
set(),
)
self.assertIn("scope_note", payload)
self.assertEqual(expected["gate"], "validity-shape")
self.assertEqual(expected["verdict"], "scoped-candidate-not-full-validity")
self.assertEqual(expected["completeness"], "blocked-scoped-subset")
self.assertEqual({cell["surface"] for cell in cells}, self.required_surfaces())
def test_at_3_1_01_third_revision_outside_the_approved_window_is_rejected_by_name(self) -> None:
"""Bounded one-factor mutation OF THE REVIEWED CANDIDATE: one extra ledger revision entry."""
payload = self.write_candidate_inventory("inventory-mutation-third-revision.json")
revisions = [entry["revision"] for entry in payload["protocol"]["revisions"]] # type: ignore[index]
self.assertEqual(revisions[:2], ["2026-07-28", "2025-11-25"])
self.assertEqual(revisions[2:], ["2025-03-26"], "only one revision entry may be added")
result = self.run_validator(["FR-13"])
combined = result.stdout + result.stderr
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertIn("2025-03-26", combined)
def test_at_3_1_03_reviewed_candidate_omission_is_rejected_by_the_triple(self) -> None:
"""Bounded one-factor mutation OF THE REVIEWED CANDIDATE: exactly one reviewed row dropped."""
payload = self.write_candidate_inventory("inventory-mutation-omission.json")
observed = {
(cell["role"], cell["kind"], cell["surface"])
for cell in payload["protocol"]["capabilities"] # type: ignore[index]
}
self.assertEqual(len(observed), 49, "only one reviewed identity may be dropped")
self.assertNotIn(("client", "method", "prompts/get"), observed)
result = self.run_validator(["FR-13"])
combined = result.stdout + result.stderr
self.assertNotEqual(result.returncode, 0, result.stdout)
lines = [line for line in combined.splitlines() if "OMISSION" in line]
self.assertTrue(
any("prompts/get" in line for line in lines),
f"the omitted triple must be printed by name: {combined}",
)
self.assertTrue(
any("client" in line and "method" in line for line in lines),
f"the full (role, kind, surface) triple must be printed: {combined}",
)
def test_at_3_1_03_undocumented_surface_is_rejected_by_the_triple(self) -> None:
"""Bounded one-factor mutation OF THE REVIEWED CANDIDATE: one extra undocumented triple."""
payload = self.write_candidate_inventory("inventory-mutation-undocumented-surface.json")
observed = {
(cell["role"], cell["kind"], cell["surface"])
for cell in payload["protocol"]["capabilities"] # type: ignore[index]
}
self.assertEqual(len(observed), 51, "only one extra identity may be added")
self.assertIn(("server", "method", "resources/read"), observed)
result = self.run_validator(["FR-13"])
combined = result.stdout + result.stderr
self.assertNotEqual(result.returncode, 0, result.stdout)
lines = [line for line in combined.splitlines() if "UNDOCUMENTED_SURFACE" in line]
self.assertTrue(
any("resources/read" in line for line in lines),
f"the undocumented triple must be printed by name: {combined}",
)
self.assertTrue(
any("server" in line and "method" in line for line in lines),
f"the full (role, kind, surface) triple must be printed: {combined}",
)
def test_at_3_1_01_revision_missing_only_the_digest_is_rejected_by_that_field(self) -> None:
"""One-factor provenance negative: only `provenance.sha256` of the modern revision is gone."""
payload = self.write_candidate_inventory("inventory-mutation-missing-digest.json")
modern = next(
entry
for entry in payload["protocol"]["revisions"] # type: ignore[index]
if entry["revision"] == "2026-07-28"
)
self.assertEqual(sorted(modern["provenance"]), ["captured_on", "locator"])
result = self.run_validator(["FR-13"])
combined = result.stdout + result.stderr
self.assertNotEqual(result.returncode, 0, result.stdout)
diagnostics = [line for line in combined.splitlines() if line.startswith("error[")]
digest_lines = [line for line in diagnostics if "sha256" in line]
self.assertTrue(
any("2026-07-28" in line for line in digest_lines),
f"the missing digest must be reported for the revision that lost it: {combined}",
)
self.assertFalse(
any("captured_on" in line for line in diagnostics),
f"the date is still present and must not be reported missing: {combined}",
)
def test_at_3_1_01_revision_missing_only_the_date_is_rejected_by_that_field(self) -> None:
"""One-factor provenance negative: only `provenance.captured_on` of the modern revision is gone."""
payload = self.write_candidate_inventory("inventory-mutation-missing-date.json")
modern = next(
entry
for entry in payload["protocol"]["revisions"] # type: ignore[index]
if entry["revision"] == "2026-07-28"
)
self.assertEqual(sorted(modern["provenance"]), ["locator", "sha256"])
result = self.run_validator(["FR-13"])
combined = result.stdout + result.stderr
self.assertNotEqual(result.returncode, 0, result.stdout)
diagnostics = [line for line in combined.splitlines() if line.startswith("error[")]
date_lines = [line for line in diagnostics if "captured_on" in line]
self.assertTrue(
any("2026-07-28" in line for line in date_lines),
f"the missing capture date must be reported for the revision that lost it: {combined}",
)
self.assertFalse(
any("sha256" in line for line in diagnostics),
f"the digest is still present and must not be reported missing: {combined}",
)
def test_at_3_1_03_ordered_validity_first_gate_table_is_exercised_when_mode_exists(self) -> None:
"""The agreed validity-first gate table (A0 / B3 / B0 / invalid1) on the real CLI.
While the CLI exposes no `--mode`, this stays an honest interface gap (skip, never a
synthetic red). As soon as `--mode` exists the same test body exercises every agreed row, so
a wrong Gate A/B implementation fails here instead of being skipped.
"""
probe = subprocess.run(
[sys.executable, str(VALIDATOR), "--help"],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
help_text = probe.stdout + probe.stderr
self.assertEqual(probe.returncode, 0, help_text)
if "--mode" not in help_text:
self.skipTest(
"interface gap, not a behavioural red: the validator CLI exposes no --mode yet, so "
"the ordered validity/feature-complete gate table cannot be exercised on this "
"revision"
)
self.install_canonical_schema()
# invalid1 - structurally invalid input: `invalid` in BOTH modes, step-0 diagnostics, and no
# feature-complete evaluation at all (accepted spec `remaining-ac-spec.md:392`).
self.inventory_path.write_text("{not-json", encoding="utf-8")
for mode in ("validity", "feature-complete"):
with self.subTest(row="structural-invalid", mode=mode):
result = self.run_validator(["FR-13"], install_schema=False, mode=mode)
combined = result.stdout + result.stderr
self.assertEqual(result.returncode, 1, combined)
self.assertIn("status=invalid", combined)
self.assertIn("error[", combined)
self.assertNotIn("status=blocked", combined)
self.assertNotIn("status=complete", combined)
self.assertNotRegex(combined, r"blockers=[1-9]")
# invalid1 - semantic validity diagnostics (only the modern `provenance.sha256` is gone):
# `invalid` in BOTH modes, the targeted missing-digest diagnostic, no feature-complete output
# (accepted spec `remaining-ac-spec.md:393`).
self.write_candidate_inventory("inventory-mutation-missing-digest.json")
for mode in ("validity", "feature-complete"):
with self.subTest(row="semantic-invalid", mode=mode):
result = self.run_validator(["FR-13"], mode=mode)
combined = result.stdout + result.stderr
self.assertEqual(result.returncode, 1, combined)
self.assertIn("status=invalid", combined)
diagnostics = [line for line in combined.splitlines() if line.startswith("error[")]
self.assertTrue(
any("2026-07-28" in line and "sha256" in line for line in diagnostics),
f"the missing digest must be named for the revision that lost it: {combined}",
)
self.assertNotIn("status=blocked", combined)
self.assertNotIn("status=complete", combined)
self.assertNotRegex(combined, r"blockers=[1-9]")
# invalid1 - the rejected false-not-applicable variant: applicable scheduled work renamed to
# `not_applicable` must stay `invalid` in BOTH modes, never `complete` with 0 blockers.
self.write_candidate_inventory("inventory-mutation-false-not-applicable.json")
for mode in ("validity", "feature-complete"):
with self.subTest(row="false-not-applicable", mode=mode):
result = self.run_validator(["FR-13"], mode=mode)
combined = result.stdout + result.stderr
self.assertEqual(result.returncode, 1, combined)
self.assertIn("status=invalid", combined)
self.assertNotIn("status=blocked", combined)
self.assertNotIn("status=complete", combined)
self.assertNotRegex(combined, r"blockers=[1-9]")
# A0 / B3 - a truthful candidate passes validity and is blocked by the feature gate.
self.write_candidate_inventory()
validity = self.run_validator(["FR-13"], mode="validity")
self.assertEqual(validity.returncode, 0, validity.stdout + validity.stderr)
self.assertIn("Capability inventory validation passed", validity.stdout + validity.stderr)
feature_complete = self.run_validator(["FR-13"], mode="feature-complete")
self.assertEqual(
feature_complete.returncode,
3,
feature_complete.stdout + feature_complete.stderr,
)
self.assertIn("status=blocked", feature_complete.stdout + feature_complete.stderr)
self.assertRegex(
feature_complete.stdout + feature_complete.stderr,
r"blockers=[1-9][0-9]*",
)
# B0 - the explicitly SYNTHETIC no-blocker input (hypothetical completed state, marked with
# its own owner) is complete in the feature gate.
self.write_candidate_inventory("inventory-synthetic-no-blocker.json")
validity = self.run_validator(["FR-13"], mode="validity")
self.assertEqual(validity.returncode, 0, validity.stdout + validity.stderr)
self.assertIn("Capability inventory validation passed", validity.stdout + validity.stderr)
feature_complete = self.run_validator(["FR-13"], mode="feature-complete")
self.assertEqual(
feature_complete.returncode,
0,
feature_complete.stdout + feature_complete.stderr,
)
self.assertIn("status=complete", feature_complete.stdout + feature_complete.stderr)
self.assertRegex(
feature_complete.stdout + feature_complete.stderr,
r"blockers=0\b",
)
# --- Story 3.1 remaining-foundation ATDD additions ------------------------------------------
# Oracle inputs: the reviewed v2 candidate (`inventory-reviewed-v2-candidate.json`), its bounded
# one-factor mutations, and the independent per-era frozen projections. The reviewed identity is
# the triple `(role, kind, surface)`; `role` is one enum {server, client, both} with a
# kind-specific meaning (declarer for capability/extension, sender for method/notification).
# The target validator behaviour (v2 protocol block, Gate A/B split, named diagnostics) does not
# exist on this revision, so the assertions below are the contract the implementation stage must
# satisfy. Anything that can only fail because a CLI flag or a module is absent is recorded as an
# honest API gap, never as a behavioural red and never as a pass.
REVIEWED_IDENTITY_TRIPLES = {
# server capabilities (surface-inventory.md section 3)
("server", "capability", "experimental"),
("server", "capability", "logging"),
("server", "capability", "completions"),
("server", "capability", "prompts"),
("server", "capability", "resources"),
("server", "capability", "tools"),
("server", "capability", "extensions"),
("server", "capability", "tasks"),
# client capabilities (section 4)
("client", "capability", "experimental"),
("client", "capability", "roots"),
("client", "capability", "sampling"),
("client", "capability", "elicitation"),
("client", "capability", "extensions"),
("client", "capability", "tasks"),
# methods (section 5); role = the sending party
("client", "method", "tools/list"),
("client", "method", "tools/call"),
("client", "method", "resources/list"),
("client", "method", "resources/read"),
("client", "method", "resources/templates/list"),
("client", "method", "resources/subscribe"),
("client", "method", "resources/unsubscribe"),
("client", "method", "prompts/list"),
("client", "method", "prompts/get"),
("client", "method", "completion/complete"),
("client", "method", "logging/setLevel"),
("client", "method", "initialize"),
("both", "method", "ping"),
("client", "method", "server/discover"),
("client", "method", "subscriptions/listen"),
("server", "method", "roots/list"),
("server", "method", "sampling/createMessage"),
("server", "method", "elicitation/create"),
("both", "method", "tasks/get"),
("both", "method", "tasks/list"),
("both", "method", "tasks/result"),
("both", "method", "tasks/cancel"),
# notifications (section 6)
("server", "notification", "notifications/tools/list_changed"),
("server", "notification", "notifications/resources/list_changed"),
("server", "notification", "notifications/prompts/list_changed"),
("server", "notification", "notifications/resources/updated"),
("server", "notification", "notifications/progress"),
("server", "notification", "notifications/message"),
("server", "notification", "notifications/subscriptions/acknowledged"),
("both", "notification", "notifications/cancelled"),
("client", "notification", "notifications/initialized"),
("client", "notification", "notifications/roots/list_changed"),
("both", "notification", "notifications/tasks/status"),
("server", "notification", "notifications/elicitation/complete"),
# extensions (section 7)
("server", "extension", "io.modelcontextprotocol/tasks"),
("client", "extension", "io.modelcontextprotocol/oauth-client-credentials"),
}
REVIEWED_ROLES = {"server", "client", "both"}
REVIEWED_KINDS = {"capability", "method", "notification", "extension"}
REVIEWED_STATUSES = {"supported", "not_applicable", "experimental", "deferred"}
# Independent per-era ABSENCE oracle, read off the "absent" cells of the reviewed tables in
# `build/remaining-ac-spec/surface-inventory.md` sections 3-7 (deliberately NOT taken from the
# fixture). Rule O2 resolves an era in two steps (`remaining-ac-spec.md:296-300`): a surface whose
# `revisions[]` does not contain the era is ABSENT for that era - never `not_applicable` - and only
# then is the shallow `per_revision[era]` merge applied. Applicable per-era completeness is
# therefore `REVIEWED_IDENTITY_TRIPLES - REVIEWED_ABSENT_<era>`, not the 50-identity default union.
REVIEWED_ABSENT_MODERN = {
# sections 3/4: modern expresses Tasks as an extension, so the server `tasks` capability is
# absent, and the client `tasks` declaration does not exist in modern either
("server", "capability", "tasks"),
("client", "capability", "tasks"),
# section 5: method literals absent from the pinned modern schema
("client", "method", "resources/subscribe"),
("client", "method", "resources/unsubscribe"),
("client", "method", "logging/setLevel"),
("client", "method", "initialize"),
("both", "method", "ping"),
("both", "method", "tasks/get"),
("both", "method", "tasks/list"),
("both", "method", "tasks/result"),
("both", "method", "tasks/cancel"),
# section 6: notifications absent from the pinned modern schema
("client", "notification", "notifications/initialized"),
("client", "notification", "notifications/roots/list_changed"),
("both", "notification", "notifications/tasks/status"),
("server", "notification", "notifications/elicitation/complete"),
}
REVIEWED_ABSENT_LEGACY = {
# sections 3/4: neither legacy `ServerCapabilities` nor legacy `ClientCapabilities` has an
# `extensions` key
("server", "capability", "extensions"),
("client", "capability", "extensions"),
# sections 5/6: modern-only method literals and the subscription acknowledgement
("client", "method", "server/discover"),
("client", "method", "subscriptions/listen"),
("server", "notification", "notifications/subscriptions/acknowledged"),
# section 7: both extension identifiers are declared in the modern schema only
("server", "extension", "io.modelcontextprotocol/tasks"),
("client", "extension", "io.modelcontextprotocol/oauth-client-credentials"),
}
def load_candidate_fixture(self, name: str = "inventory-reviewed-v2-candidate.json") -> dict[str, object]:
return self.load_revision_fixture(name)
def write_candidate_inventory(self, name: str = "inventory-reviewed-v2-candidate.json") -> dict[str, object]:
"""Install a reviewed candidate/mutation fixture as the validator input."""
payload = self.load_candidate_fixture(name)
for flow in payload.get("flows", []): # type: ignore[union-attr]
for kind in ("automated", "manual"):
for relative in flow.get("evidence", {}).get(kind, []):
self.write_evidence(relative)
self.inventory_path.write_text(
json.dumps(payload, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return payload
def effective_entry(self, entry: dict[str, object], revision: str) -> dict[str, object]:
merged = {key: value for key, value in entry.items() if key != "per_revision"}
overrides = entry.get("per_revision", {}).get(revision, {}) # type: ignore[union-attr]
merged.update(overrides)
return merged
def test_at_3_1_03_reviewed_candidate_is_identity_complete_and_metadata_stays_separate(self) -> None:
"""The reviewed completeness basis itself: one judged entry per reviewed row, no more, no less."""
payload = self.load_candidate_fixture()
protocol: Any = payload["protocol"]
self.assertEqual(payload["schema_version"], 2)
self.assertEqual(
[entry["revision"] for entry in protocol["revisions"]],
["2026-07-28", "2025-11-25"],
)
for entry in protocol["revisions"]:
provenance = entry["provenance"]
self.assertTrue(provenance["locator"] and provenance["sha256"] and provenance["captured_on"])
self.assertEqual(protocol["withdrawn"], ["2025-06-18", "2025-03-26"])
capabilities = protocol["capabilities"]
triples = [(cell["role"], cell["kind"], cell["surface"]) for cell in capabilities]
self.assertEqual(len(triples), len(set(triples)), "identity is the (role, kind, surface) triple")
self.assertEqual(set(triples), self.REVIEWED_IDENTITY_TRIPLES)
for cell in capabilities:
self.assertIn(cell["role"], self.REVIEWED_ROLES)
self.assertIn(cell["kind"], self.REVIEWED_KINDS)
self.assertIn(cell["status"], self.REVIEWED_STATUSES)
self.assertTrue(cell["owner"] and cell["fallback"] and cell["evidence_target"])
if cell["status"] == "supported":
self.assertNotIn("reason", cell)
else:
self.assertTrue(cell["reason"], cell)
# `io.modelcontextprotocol/related-task` is a non-advertisable `_meta` metadata note, not a
# capability extension: it must stay out of the four-kind ledger and out of the advertisement.
self.assertFalse(
any(
cell["surface"] == "io.modelcontextprotocol/related-task"
for cell in capabilities
),
"related-task must not be recorded as a capability/extension entry",
)
# Per-revision overrides are resolved with a shallow per-field merge (rule O2).
progress = next(
cell
for cell in capabilities
if cell["surface"] == "notifications/progress" and cell["kind"] == "notification"
)
self.assertEqual(self.effective_entry(progress, "2025-11-25")["role"], "both")
self.assertEqual(self.effective_entry(progress, "2026-07-28")["role"], "server")
roots = next(
cell
for cell in capabilities
if cell["surface"] == "roots" and cell["kind"] == "capability"
)
self.assertEqual(self.effective_entry(roots, "2026-07-28")["subfeatures"], [])
self.assertEqual(self.effective_entry(roots, "2025-11-25")["subfeatures"], ["listChanged"])
# A `both` row satisfies both sender sides but is never advertisable.
tools = next(cell for cell in capabilities if cell["surface"] == "tools" and cell["role"] == "server")
self.assertEqual(tools["status"], "supported")
self.assertEqual(tools["advertised"], {"tools": {"listChanged": False}})
for cell in capabilities:
resolved = [self.effective_entry(cell, revision) for revision in ("2026-07-28", "2025-11-25")]
for effective in resolved:
if effective["role"] != "server" or effective["kind"] not in {"capability", "extension"}:
self.assertIsNone(effective["advertised"], cell["surface"])
if effective["kind"] in {"method", "notification"}:
self.assertIsNone(effective["advertised"], cell["surface"])
def test_at_3_1_03_duplicate_surface_identity_is_rejected_by_the_triple(self) -> None:
self.write_candidate_inventory("inventory-mutation-duplicate-identity.json")
result = self.run_validator(["FR-13"])
combined = result.stdout + result.stderr
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertIn("DUPLICATE_SURFACE_IDENTITY", combined)
self.assertIn("experimental", combined)
def test_at_3_1_03_advertised_fragment_on_a_non_advertisable_role_is_rejected(self) -> None:
self.write_candidate_inventory("inventory-mutation-advertised-on-client-role.json")
result = self.run_validator(["FR-13"])
combined = result.stdout + result.stderr
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertIn("ADVERTISED_ON_NON_ADVERTISABLE", combined)
self.assertIn("elicitation", combined)
def test_at_3_1_03_supported_server_capability_must_carry_an_advertised_fragment(self) -> None:
self.write_candidate_inventory("inventory-mutation-supported-server-without-advertised.json")
result = self.run_validator(["FR-13"])
combined = result.stdout + result.stderr
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertIn("tools", combined)
self.assertIn("advertised", combined.lower())
def test_at_3_1_03_effective_per_era_identities_follow_eligibility_then_o2(self) -> None:
"""O2 coverage beyond the default 50 identities: effective per-era identity sets.
Rule O2 resolves an era in two steps (accepted spec `remaining-ac-spec.md:296-300`): a surface
whose `revisions[]` does not contain the era is ABSENT for that era - never `not_applicable` -
and only then is the shallow `per_revision[era]` merge applied. The expectations below are the
reviewed absence sets read off `surface-inventory.md` sections 3-7, not a copy of the fixture:
the applicable set of an era is `REVIEWED_IDENTITY_TRIPLES - REVIEWED_ABSENT_<era>` (35 modern,
43 legacy on this candidate). The 50-identity default union stays a separate ORACLE-INPUT and is
never reported as per-era completeness.
"""
payload = self.load_candidate_fixture()
protocol: Any = payload["protocol"]
capabilities: Any = protocol["capabilities"]
per_era: dict[str, list[tuple[Any, Any, Any]]] = {}
progress_reviewed_default = ("server", "notification", "notifications/progress")
# The one reviewed per-era identity substitution (surface-inventory.md section 2.1/6): the
# progress sender is `server` in the modern revision and `both` in the legacy revision.
reviewed_progress_per_era = {
"2026-07-28": ("server", "notification", "notifications/progress"),
"2025-11-25": ("both", "notification", "notifications/progress"),
}
for era, absent in (
("2026-07-28", self.REVIEWED_ABSENT_MODERN),
("2025-11-25", self.REVIEWED_ABSENT_LEGACY),
):
with self.subTest(era=era):
eligible = [cell for cell in capabilities if era in cell["revisions"]]
triples = [
(self.effective_entry(cell, era)["role"], cell["kind"], cell["surface"])
for cell in eligible
]
per_era[era] = triples
expected = (
self.REVIEWED_IDENTITY_TRIPLES
- absent
- {progress_reviewed_default}
) | {reviewed_progress_per_era[era]}
self.assertEqual(
len(triples),
len(set(triples)),
"an override must not create a duplicate EFFECTIVE identity",
)
self.assertEqual(
set(triples),
expected,
f"the applicable identities of {era} must match the reviewed tables",
)
self.assertEqual(
sorted(absent),
sorted(
(cell["role"], cell["kind"], cell["surface"])
for cell in capabilities
if era not in cell["revisions"]
),
"every explicitly absent reviewed identity must be ineligible in that era",
)
self.assertEqual(
set(triples) & absent,
set(),
"an absent identity can never be part of the applicable set",
)
self.assertEqual(len(per_era["2026-07-28"]), 35)
self.assertEqual(len(per_era["2025-11-25"]), 43)
for era, triples in per_era.items():
self.assertNotEqual(
set(triples),
self.REVIEWED_IDENTITY_TRIPLES,
f"the default 50-identity union is not the applicable set of {era}",
)
# The only identity that resolves to a different ROLE per era is the agreed progress override.
overridden = set()
for cell in capabilities:
if "2026-07-28" in cell["revisions"] and "2025-11-25" in cell["revisions"]:
roles = {
era: self.effective_entry(cell, era)["role"]
for era in ("2026-07-28", "2025-11-25")
}
if roles["2026-07-28"] != roles["2025-11-25"]:
overridden.add((cell["role"], cell["kind"], cell["surface"]))
self.assertEqual(
overridden,
{("server", "notification", "notifications/progress")},
"the agreed per-era role difference is exactly the progress sender override",
)
progress = next(
cell
for cell in capabilities
if cell["surface"] == "notifications/progress" and cell["kind"] == "notification"
)
self.assertEqual(progress["role"], "server")
self.assertEqual(progress["per_revision"]["2025-11-25"]["role"], "both")
self.assertEqual(self.effective_entry(progress, "2026-07-28")["role"], "server")
self.assertEqual(self.effective_entry(progress, "2025-11-25")["role"], "both")
# --- B0 / invalid1 inputs of the agreed Gate table (round-2 corrections) ----------------------
# Both are fixture oracles only: they inspect the input documents and never run the validator, so
# nothing here is called validator GREEN. The executable Gate rows live in the `--mode` table above
# and stay an honest interface gap until the CLI exposes `--mode`.
def test_at_3_1_03_synthetic_no_blocker_variant_is_a_hypothetical_completed_state(self) -> None:
"""`inventory-synthetic-no-blocker.json` (B0 row) is synthetic, never a status relabel.
Every `deferred` reviewed surface is resolved to the hypothetical completed state
(`supported`), never to `not_applicable`; advertisable server capability/extension rows carry
the exact `advertised` fragment their reviewed `subfeatures` declare; and every flipped row is
marked with the owner `atdd-synthetic-no-blocker-input`. The document therefore contains no
deferral at all - a no-blocker input under the agreed gate reading and under any stricter one.
"""
payload = self.load_candidate_fixture("inventory-synthetic-no-blocker.json")
protocol: Any = payload["protocol"]
capabilities: Any = protocol["capabilities"]
reviewed_payload: dict[str, Any] = self.load_candidate_fixture()
reviewed: Any = reviewed_payload["protocol"]["capabilities"]
reviewed_by_identity = {
(cell["role"], cell["kind"], cell["surface"]): cell for cell in reviewed
}
self.assertEqual(
[cell["surface"] for cell in capabilities if cell["status"] == "deferred"],
[],
"the no-blocker input must not defer any reviewed surface",
)
self.assertEqual(
sorted(cell["surface"] for cell in capabilities if cell["status"] == "not_applicable"),
sorted(cell["surface"] for cell in reviewed if cell["status"] == "not_applicable"),
"the reviewed `not_applicable` rows must stay exactly as reviewed",
)
flipped = 0
for cell in capabilities:
original = reviewed_by_identity[(cell["role"], cell["kind"], cell["surface"])]
if original["status"] != "deferred":
self.assertEqual(
cell,
original,
f"only deferred rows may differ from the reviewed candidate: {cell['surface']}",
)
continue
flipped += 1
self.assertEqual(cell["status"], "supported", cell["surface"])
self.assertNotIn("reason", cell, "a hypothetical `supported` row carries no reason")
self.assertEqual(cell["owner"], "atdd-synthetic-no-blocker-input", cell["surface"])
self.assertEqual(
sorted(cell),
sorted(set(original) - {"reason"}),
"a flipped row may only change status/reason/advertised/open_subfeatures",
)
self.assertEqual(flipped, 26, "every deferred reviewed row must be flipped")
for cell in capabilities:
if cell["owner"].startswith("atdd-synthetic") and cell["role"] == "server" and cell[
"kind"
] in {"capability", "extension"}:
self.assertIsNotNone(
cell["advertised"],
f"a supported server declaration must advertise: {cell['surface']}",
)
def test_at_3_1_03_false_not_applicable_input_keeps_the_planned_reasons(self) -> None:
"""`inventory-mutation-false-not-applicable.json` (invalid1 row) is the REJECTED predecessor.
It is the variant that renames every `deferred` status to `not_applicable` while keeping the
planned/deferred reasons, i.e. it claims that applicable, scheduled work is inapplicable. A
truthful validity gate must answer `invalid` in BOTH modes and never `complete` / 0 blockers;
the executable assertion lives in the `--mode` table, which is an honest interface gap until
`--mode` exists.
"""
payload = self.load_candidate_fixture("inventory-mutation-false-not-applicable.json")
protocol: Any = payload["protocol"]
capabilities: Any = protocol["capabilities"]
reviewed_payload: dict[str, Any] = self.load_candidate_fixture()
reviewed: Any = reviewed_payload["protocol"]["capabilities"]
reviewed_by_identity = {
(cell["role"], cell["kind"], cell["surface"]): cell for cell in reviewed
}
relabelled = 0
falsely_inapplicable_server_capabilities = []
for cell in capabilities:
original = reviewed_by_identity[(cell["role"], cell["kind"], cell["surface"])]
if original["status"] != "deferred":
self.assertEqual(cell, original, f"only deferred rows may be relabelled: {cell['surface']}")
continue
relabelled += 1
self.assertEqual(cell["status"], "not_applicable", cell["surface"])
self.assertEqual(
cell["reason"],
original["reason"],
"the relabel keeps the planned/deferred reason - that is the false claim",
)
if cell["role"] == "server" and cell["kind"] == "capability":
falsely_inapplicable_server_capabilities.append(cell["surface"])
self.assertEqual(relabelled, 26, "every deferred reviewed row must be relabelled")
self.assertEqual(
sorted(falsely_inapplicable_server_capabilities),
["completions", "logging", "prompts", "tasks"],
"applicable server capabilities must never be declared inapplicable",
)
if __name__ == "__main__":
unittest.main()