CI / Rust Checks (pull_request) Failing after 13m41s
CI / UI Checks (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Deploy (pull_request) Skipped
Community Release / release (push) Failing after 13m33s
Recovery delivery of the accumulated, TL-accepted Epic 3 work that had never been
committed to Git (stories 3.6-3.17, see the crank board). Story 3.18 is not accepted
yet and is deliberately excluded: its logging surface, its capability registration,
its oracle projections and its tests are not part of this tree.
Accepted scope
- 3.6 typed static Resource Drafts; 3.7 publication and exact binding of static
Resource Versions; 3.8 MCP discovery/read and revocation of static Resources;
- 3.9 Resource import/export in API and UI plus TLS dependency maintenance;
3.10 REST-backed Resource templates; 3.11 shared REST read runtime and the MCP
revision/capability foundation;
- 3.12 parameterised Prompt Drafts with preview; 3.13 Prompt publication and exact
version binding to Agents; 3.14 MCP prompts; 3.15 Prompt import/export;
- 3.16 composite Prompts with published Resources (dependency content scan plus
dynamic read through the accepted 3.11 runtime, admin preview and MCP prompts/get);
- 3.17 bounded MCP completion suggestions from typed published schemas and the
resources/list admission fix for bound template publications.
Accepted evidence being reused (unchanged code paths)
- 3.16 candidate: core 7, admin-api 22, registry 4, mcp-server 4, UI 6; one final
`just verify` run 1651 passed / 0 failed / 2 ignored.
- 3.17 candidate: full run 1672 passed / 0 failed / 2 ignored, QA targeted reruns and
a real MCP Inspector 2.12.0 client smoke.
Delivery engineering in this commit
- CI: one event per change (feature branches are gated by their pull request, push
runs only on main), story events run the changed packages, the full workspace suite
stays the feature/release gate; no cache/sharding without a measured runner check.
- Release: the packaged archives, checksums and SBOM are now uploaded to the Gitea
release entry of the tag instead of dying with the runner workspace.
Not included on purpose: Story 3.18 work, build caches, runtime databases, local
credentials and scratch evidence.
Refs: crank kanban t_e39d7263; baseline 1b6db09bd1.
1587 lines
74 KiB
Python
1587 lines
74 KiB
Python
import copy
|
|
import importlib.util
|
|
import json
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
VALIDATOR = ROOT / "scripts" / "validate-capability-inventory.py"
|
|
CANONICAL_INVENTORY = ROOT / "docs" / "capability-inventory.json"
|
|
CANONICAL_SCHEMA = ROOT / "docs" / "schemas" / "capability-inventory.schema.json"
|
|
COMMUNITY_CHECKER = ROOT / "scripts" / "check-community-scope.py"
|
|
COMMUNITY_README = ROOT / "README.md"
|
|
DOCS_INDEX = ROOT / "docs" / "README.md"
|
|
COMMUNITY_INTRO = ROOT / "docs" / "intro.md"
|
|
COMMUNITY_README_EN = ROOT / "docs" / "en" / "README.md"
|
|
JUSTFILE = ROOT / "justfile"
|
|
CI_WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
|
|
RELEASE_WORKFLOW = ROOT / ".gitea" / "workflows" / "release.yml"
|
|
REQUIRED_SURFACES = ROOT / "docs" / "capability-baseline" / "required-surfaces.json"
|
|
MCP_REVISION_FIXTURES = ROOT / "tests" / "fixtures" / "mcp-revisions"
|
|
|
|
|
|
def load_validator_module():
|
|
spec = importlib.util.spec_from_file_location("capability_inventory_validator", VALIDATOR)
|
|
if spec is None or spec.loader is None:
|
|
raise RuntimeError("cannot load capability inventory validator")
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
class CapabilityInventoryValidatorTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self._temporary_directory = tempfile.TemporaryDirectory()
|
|
self.root = Path(self._temporary_directory.name)
|
|
self.inventory_path = self.root / "docs" / "capability-inventory.json"
|
|
self.schema_path = (
|
|
self.root / "docs" / "schemas" / "capability-inventory.schema.json"
|
|
)
|
|
self.inventory_path.parent.mkdir(parents=True)
|
|
self.schema_path.parent.mkdir(parents=True)
|
|
|
|
def tearDown(self) -> None:
|
|
self._temporary_directory.cleanup()
|
|
|
|
def write_evidence(self, relative_path: str, content: str = "verified\n") -> str:
|
|
path = self.root / relative_path
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(content, encoding="utf-8")
|
|
return relative_path
|
|
|
|
def valid_flow(
|
|
self,
|
|
flow_id: str,
|
|
flow_type: str,
|
|
requirement: str,
|
|
status: str = "implemented",
|
|
capabilities: list[str] | None = None,
|
|
) -> dict[str, object]:
|
|
automated = self.write_evidence(
|
|
f"evidence/automated/{flow_id}.txt",
|
|
f"{flow_id}: automated acceptance evidence\n",
|
|
)
|
|
manual = self.write_evidence(
|
|
f"evidence/manual/{flow_id}.md",
|
|
f"# {flow_id}\n\nManual verification complete.\n",
|
|
)
|
|
return {
|
|
"id": flow_id,
|
|
"type": flow_type,
|
|
"requirements": [requirement],
|
|
"user_outcome": f"User completes {flow_id.replace('-', ' ')}.",
|
|
"owner": "community-release",
|
|
"status": status,
|
|
"capabilities": capabilities or ["tools"],
|
|
"evidence": {
|
|
"automated": [automated],
|
|
"manual": [manual],
|
|
},
|
|
}
|
|
|
|
def write_inventory(self, flows: list[dict[str, object]]) -> None:
|
|
payload = {
|
|
"schema_version": 1,
|
|
"product": "crank-community",
|
|
"flows": flows,
|
|
}
|
|
self.inventory_path.write_text(
|
|
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
def install_canonical_schema(self) -> None:
|
|
self.assertTrue(
|
|
CANONICAL_SCHEMA.is_file(),
|
|
f"Story 1.1 must create {CANONICAL_SCHEMA.relative_to(ROOT)}",
|
|
)
|
|
shutil.copyfile(CANONICAL_SCHEMA, self.schema_path)
|
|
|
|
def run_validator(
|
|
self,
|
|
required_frs: list[str],
|
|
install_schema: bool = True,
|
|
mode: str | None = None,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
if install_schema:
|
|
self.install_canonical_schema()
|
|
command = [
|
|
sys.executable,
|
|
str(VALIDATOR),
|
|
"--root",
|
|
str(self.root),
|
|
"--inventory",
|
|
str(self.inventory_path.relative_to(self.root)),
|
|
"--schema",
|
|
str(self.schema_path.relative_to(self.root)),
|
|
]
|
|
for requirement in required_frs:
|
|
command.extend(["--required-fr", requirement])
|
|
if mode is not None:
|
|
command.extend(["--mode", mode])
|
|
return subprocess.run(
|
|
command,
|
|
check=False,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
def test_accepts_complete_inventory_with_all_flow_types_and_statuses(self) -> None:
|
|
flows = [
|
|
self.valid_flow("ui-create-method", "ui", "FR-1", "implemented"),
|
|
self.valid_flow("api-create-method", "api", "FR-2", "planned"),
|
|
self.valid_flow("mcp-tool-call", "mcp", "FR-3", "gap"),
|
|
self.valid_flow("mcp-resource-read", "mcp", "FR-4", "blocked"),
|
|
]
|
|
self.write_inventory(flows)
|
|
|
|
result = self.run_validator(["FR-1", "FR-2", "FR-3", "FR-4"])
|
|
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertIn("Capability inventory validation passed", result.stdout)
|
|
self.assertRegex(result.stdout, r"\btotal=4\b")
|
|
self.assertRegex(result.stdout, r"\bimplemented=1\b")
|
|
self.assertRegex(result.stdout, r"\bplanned=1\b")
|
|
self.assertRegex(result.stdout, r"\bgap=1\b")
|
|
self.assertRegex(result.stdout, r"\bblocked=1\b")
|
|
|
|
def test_counts_only_implemented_flows_as_pass(self) -> None:
|
|
flows = [
|
|
self.valid_flow("implemented-flow", "api", "FR-1", "implemented"),
|
|
self.valid_flow("planned-flow", "ui", "FR-2", "planned"),
|
|
self.valid_flow("gap-flow", "mcp", "FR-3", "gap"),
|
|
self.valid_flow("blocked-flow", "mcp", "FR-4", "blocked"),
|
|
]
|
|
self.write_inventory(flows)
|
|
|
|
result = self.run_validator(["FR-1", "FR-2", "FR-3", "FR-4"])
|
|
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertRegex(result.stdout, r"\bpass=1\b")
|
|
self.assertNotRegex(result.stdout, r"\bpass=4\b")
|
|
|
|
def test_rejects_duplicate_id_unknown_enum_and_missing_required_fields(self) -> None:
|
|
base = self.valid_flow("flow-a", "api", "FR-1")
|
|
cases: list[tuple[str, list[dict[str, object]], str]] = []
|
|
|
|
duplicate = copy.deepcopy(base)
|
|
duplicate["user_outcome"] = "A distinct outcome with a duplicate identifier."
|
|
cases.append(("duplicate ID", [base, duplicate], "DUPLICATE_FLOW_ID"))
|
|
|
|
unknown_status = copy.deepcopy(base)
|
|
unknown_status["status"] = "done"
|
|
cases.append(("unknown status", [unknown_status], "UNKNOWN_STATUS"))
|
|
|
|
unknown_type = copy.deepcopy(base)
|
|
unknown_type["type"] = "worker"
|
|
cases.append(("unknown type", [unknown_type], "UNKNOWN_FLOW_TYPE"))
|
|
|
|
no_owner = copy.deepcopy(base)
|
|
no_owner.pop("owner")
|
|
cases.append(("missing owner", [no_owner], "MISSING_OWNER"))
|
|
|
|
no_outcome = copy.deepcopy(base)
|
|
no_outcome.pop("user_outcome")
|
|
cases.append(("missing required field", [no_outcome], "MISSING_REQUIRED_FIELD"))
|
|
|
|
for label, flows, expected_code in cases:
|
|
with self.subTest(label=label):
|
|
self.write_inventory(flows)
|
|
result = self.run_validator(["FR-1"])
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn(f"error[{expected_code}]", result.stderr)
|
|
|
|
def test_rejects_missing_mandatory_requirement(self) -> None:
|
|
self.write_inventory([self.valid_flow("only-fr-one", "api", "FR-1")])
|
|
|
|
result = self.run_validator(["FR-1", "FR-46"])
|
|
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("error[MISSING_REQUIRED_FR]", result.stderr)
|
|
self.assertIn("FR-46", result.stderr)
|
|
|
|
def test_rejects_missing_non_regular_and_escaping_evidence_links(self) -> None:
|
|
base = self.valid_flow("evidence-flow", "mcp", "FR-46")
|
|
outside = self.root.parent / f"{self.root.name}-outside.txt"
|
|
outside.write_text("outside\n", encoding="utf-8")
|
|
self.addCleanup(outside.unlink, missing_ok=True)
|
|
internal_target = self.write_evidence("evidence/automated/internal-target.txt")
|
|
internal_link = self.root / "evidence" / "automated" / "internal-link.txt"
|
|
internal_link.symlink_to(self.root / internal_target)
|
|
outside_link = self.root / "evidence" / "automated" / "outside-link.txt"
|
|
outside_link.symlink_to(outside)
|
|
broken_cases = [
|
|
"evidence/automated/does-not-exist.txt",
|
|
"evidence/automated",
|
|
f"../{outside.name}",
|
|
str(outside),
|
|
"evidence/automated/internal-link.txt",
|
|
"evidence/automated/outside-link.txt",
|
|
]
|
|
|
|
for evidence_path in broken_cases:
|
|
with self.subTest(evidence_path=evidence_path):
|
|
flow = copy.deepcopy(base)
|
|
flow["evidence"] = {
|
|
"automated": [evidence_path],
|
|
"manual": ["evidence/manual/evidence-flow.md"],
|
|
}
|
|
self.write_inventory([flow])
|
|
result = self.run_validator(["FR-46"])
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("error[BROKEN_EVIDENCE_LINK]", result.stderr)
|
|
|
|
def test_report_is_deterministic_bounded_sorted_and_redacted(self) -> None:
|
|
secret_canary = "SECRET_CANARY_sk-live-4hQ9Vw8x"
|
|
flows = []
|
|
for index in range(1_200, 0, -1):
|
|
flow = self.valid_flow(
|
|
f"invalid-flow-{index:04d}",
|
|
"api",
|
|
f"FR-{index}",
|
|
)
|
|
flow["status"] = "unknown"
|
|
flow["user_outcome"] = f"{secret_canary}:{index}"
|
|
flows.append(flow)
|
|
self.write_inventory(flows)
|
|
raw_inventory = self.inventory_path.read_text(encoding="utf-8")
|
|
|
|
first = self.run_validator(["FR-46"])
|
|
second = self.run_validator(["FR-46"])
|
|
|
|
self.assertNotEqual(first.returncode, 0)
|
|
self.assertEqual(first.stderr, second.stderr)
|
|
self.assertLessEqual(len(first.stderr.encode("utf-8")), 64 * 1024)
|
|
self.assertNotIn(secret_canary, first.stderr)
|
|
self.assertNotIn(raw_inventory, first.stderr)
|
|
diagnostics = re.findall(r"error\[([^]]+)\] ([^:]+):", first.stderr)
|
|
self.assertTrue(diagnostics)
|
|
self.assertEqual(diagnostics[:-1], sorted(diagnostics[:-1]))
|
|
self.assertEqual(diagnostics[-1][0], "TRUNCATED")
|
|
self.assertNotRegex(first.stderr, r"invalid-flow-\d{4}")
|
|
|
|
def test_rejects_malformed_documents_and_schema_contract_drift(self) -> None:
|
|
self.write_inventory([self.valid_flow("valid-flow", "api", "FR-46")])
|
|
|
|
self.inventory_path.write_text("{not-json", encoding="utf-8")
|
|
malformed_inventory = self.run_validator(["FR-46"])
|
|
self.assertNotEqual(malformed_inventory.returncode, 0)
|
|
self.assertIn("error[INVALID_JSON]", malformed_inventory.stderr)
|
|
|
|
self.write_inventory([self.valid_flow("valid-flow", "api", "FR-46")])
|
|
self.schema_path.write_text("{not-json", encoding="utf-8")
|
|
malformed_schema = self.run_validator(["FR-46"], install_schema=False)
|
|
self.assertNotEqual(malformed_schema.returncode, 0)
|
|
self.assertIn("error[INVALID_SCHEMA]", malformed_schema.stderr)
|
|
|
|
self.install_canonical_schema()
|
|
schema = json.loads(self.schema_path.read_text(encoding="utf-8"))
|
|
schema["$schema"] = "https://json-schema.org/draft/2019-09/schema"
|
|
self.schema_path.write_text(json.dumps(schema), encoding="utf-8")
|
|
drifted_schema = self.run_validator(["FR-46"], install_schema=False)
|
|
self.assertNotEqual(drifted_schema.returncode, 0)
|
|
self.assertIn("error[INVALID_SCHEMA_CONTRACT]", drifted_schema.stderr)
|
|
|
|
def test_rejects_duplicate_nonstandard_recursive_and_large_integer_json(self) -> None:
|
|
malformed_documents = {
|
|
"duplicate member": (
|
|
'{"schema_version":1,"schema_version":1,'
|
|
'"product":"crank-community","flows":[]}',
|
|
"INVALID_JSON",
|
|
),
|
|
"non-standard number": (
|
|
'{"schema_version":1,"product":NaN,"flows":[]}',
|
|
"INVALID_JSON",
|
|
),
|
|
"recursive": ("[" * 100_000 + "0" + "]" * 100_000, "INVALID_JSON"),
|
|
"large integer": ("9" * 5_000, "INVALID_JSON"),
|
|
}
|
|
|
|
for label, (content, expected_code) in malformed_documents.items():
|
|
with self.subTest(label=label):
|
|
self.inventory_path.write_text(content, encoding="utf-8")
|
|
result = self.run_validator(["FR-46"])
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn(f"error[{expected_code}]", result.stderr)
|
|
self.assertNotIn("Traceback", result.stderr)
|
|
self.assertLessEqual(len(result.stderr.encode("utf-8")), 64 * 1024)
|
|
|
|
def test_schema_parity_rejects_required_type_ref_and_pattern_drift(self) -> None:
|
|
self.write_inventory([self.valid_flow("schema-parity", "api", "FR-46")])
|
|
mutations = {
|
|
"root required missing": lambda schema: schema.pop("required"),
|
|
"root contract inversion": lambda schema: schema.update({"not": {}}),
|
|
"malformed evidence required": lambda schema: schema["$defs"]["evidence"].update(
|
|
{"required": [{}]}
|
|
),
|
|
"flow ref drift": lambda schema: schema["properties"]["flows"]["items"].update(
|
|
{"$ref": "#/$defs/evidence"}
|
|
),
|
|
"owner type drift": lambda schema: schema["$defs"]["flow"]["properties"][
|
|
"owner"
|
|
].update({"type": "integer"}),
|
|
"owner const drift": lambda schema: schema["$defs"]["flow"]["properties"][
|
|
"owner"
|
|
].update({"const": "impossible"}),
|
|
"nested applicator drift": lambda schema: schema["$defs"]["evidencePaths"][
|
|
"items"
|
|
].update({"anyOf": [{"type": "integer"}]}),
|
|
"evidence pattern drift": lambda schema: schema["$defs"]["evidencePaths"][
|
|
"items"
|
|
].update({"pattern": ".+"}),
|
|
}
|
|
|
|
for label, mutate in mutations.items():
|
|
with self.subTest(label=label):
|
|
self.install_canonical_schema()
|
|
schema = json.loads(self.schema_path.read_text(encoding="utf-8"))
|
|
mutate(schema)
|
|
self.schema_path.write_text(json.dumps(schema), encoding="utf-8")
|
|
result = self.run_validator(["FR-46"], install_schema=False)
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("error[INVALID_SCHEMA_CONTRACT]", result.stderr)
|
|
self.assertNotIn("Traceback", result.stderr)
|
|
|
|
self.install_canonical_schema()
|
|
schema = json.loads(self.schema_path.read_text(encoding="utf-8"))
|
|
schema["$defs"]["flow"]["properties"]["status"]["enum"].reverse()
|
|
self.schema_path.write_text(json.dumps(schema), encoding="utf-8")
|
|
reordered_enum = self.run_validator(["FR-46"], install_schema=False)
|
|
self.assertEqual(reordered_enum.returncode, 0, reordered_enum.stderr)
|
|
|
|
def test_rejects_boolean_schema_version_and_whitespace_required_text(self) -> None:
|
|
base = self.valid_flow("strict-scalars", "api", "FR-46")
|
|
payload_cases: list[tuple[str, dict[str, object], str]] = []
|
|
|
|
boolean_version = {
|
|
"schema_version": True,
|
|
"product": "crank-community",
|
|
"flows": [copy.deepcopy(base)],
|
|
}
|
|
payload_cases.append(("boolean inventory version", boolean_version, "MISSING_REQUIRED_FIELD"))
|
|
|
|
whitespace_owner = copy.deepcopy(base)
|
|
whitespace_owner["owner"] = " \t\n "
|
|
payload_cases.append(
|
|
(
|
|
"whitespace owner",
|
|
{"schema_version": 1, "product": "crank-community", "flows": [whitespace_owner]},
|
|
"MISSING_OWNER",
|
|
)
|
|
)
|
|
|
|
whitespace_outcome = copy.deepcopy(base)
|
|
whitespace_outcome["user_outcome"] = " \t\n "
|
|
payload_cases.append(
|
|
(
|
|
"whitespace outcome",
|
|
{"schema_version": 1, "product": "crank-community", "flows": [whitespace_outcome]},
|
|
"LIMIT_EXCEEDED",
|
|
)
|
|
)
|
|
|
|
for label, payload, expected_code in payload_cases:
|
|
with self.subTest(label=label):
|
|
self.inventory_path.write_text(json.dumps(payload), encoding="utf-8")
|
|
result = self.run_validator(["FR-46"])
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn(f"error[{expected_code}]", result.stderr)
|
|
|
|
self.write_inventory([base])
|
|
self.install_canonical_schema()
|
|
schema = json.loads(self.schema_path.read_text(encoding="utf-8"))
|
|
schema["properties"]["schema_version"]["const"] = True
|
|
self.schema_path.write_text(json.dumps(schema), encoding="utf-8")
|
|
schema_result = self.run_validator(["FR-46"], install_schema=False)
|
|
self.assertNotEqual(schema_result.returncode, 0)
|
|
self.assertIn("error[INVALID_SCHEMA_CONTRACT]", schema_result.stderr)
|
|
|
|
def test_diagnostic_accumulation_is_capped_counted_and_redacted(self) -> None:
|
|
secret_canary = "SECRET_CANARY_unknown-field"
|
|
flow = self.valid_flow("bounded-diagnostics", "api", "FR-46")
|
|
for index in range(2_000):
|
|
flow[f"{secret_canary}-{index:04d}"] = True
|
|
self.write_inventory([flow])
|
|
|
|
result = self.run_validator(["FR-46"])
|
|
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("error[TRUNCATED] /: diagnostics omitted=1000", result.stderr)
|
|
self.assertEqual(result.stderr.count("error[UNKNOWN_FIELD]"), 1_000)
|
|
self.assertNotIn(secret_canary, result.stderr)
|
|
self.assertLessEqual(len(result.stderr.encode("utf-8")), 64 * 1024)
|
|
|
|
def test_rejects_oversized_document_and_field_limits(self) -> None:
|
|
self.inventory_path.write_bytes(b" " * (4_194_304 + 1))
|
|
oversized = self.run_validator(["FR-46"])
|
|
self.assertNotEqual(oversized.returncode, 0)
|
|
self.assertIn("error[INPUT_TOO_LARGE]", oversized.stderr)
|
|
|
|
flow = self.valid_flow("bounded-flow", "api", "FR-46")
|
|
flow["user_outcome"] = "x" * 4_097
|
|
self.write_inventory([flow])
|
|
overlong = self.run_validator(["FR-46"])
|
|
self.assertNotEqual(overlong.returncode, 0)
|
|
self.assertIn("error[LIMIT_EXCEEDED]", overlong.stderr)
|
|
|
|
def test_named_string_limits_cover_below_at_and_above_boundaries(self) -> None:
|
|
string_cases = [
|
|
("id", 128, lambda length: "a" * length),
|
|
("owner", 256, lambda length: "o" * length),
|
|
("user_outcome", 4_096, lambda length: "u" * length),
|
|
("notes", 4_096, lambda length: "n" * length),
|
|
("requirements", 128, lambda length: "FR-" + "1" * (length - 3)),
|
|
]
|
|
|
|
for field, limit, make_value in string_cases:
|
|
for delta in (-1, 0, 1):
|
|
with self.subTest(field=field, length=limit + delta):
|
|
safe_field = field.replace("_", "-")
|
|
flow = self.valid_flow(f"bounded-{safe_field}-{delta + 1}", "api", "FR-46")
|
|
value = make_value(limit + delta)
|
|
if field == "requirements":
|
|
flow[field] = [value]
|
|
required_frs = [value]
|
|
else:
|
|
flow[field] = value
|
|
required_frs = ["FR-46"]
|
|
self.write_inventory([flow])
|
|
result = self.run_validator(required_frs)
|
|
if delta <= 0:
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
else:
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("error[LIMIT_EXCEEDED]", result.stderr)
|
|
|
|
def test_collection_limits_cover_below_at_and_above_boundaries(self) -> None:
|
|
module = load_validator_module()
|
|
automated = self.write_evidence("evidence/shared/automated.txt")
|
|
manual = self.write_evidence("evidence/shared/manual.md")
|
|
|
|
def direct_flow(index: int) -> dict[str, object]:
|
|
return {
|
|
"id": f"flow-{index:05d}",
|
|
"type": "api",
|
|
"requirements": ["FR-46"],
|
|
"user_outcome": "A bounded flow succeeds.",
|
|
"owner": "community-release",
|
|
"status": "implemented",
|
|
"capabilities": ["tools"],
|
|
"evidence": {"automated": [automated], "manual": [manual]},
|
|
}
|
|
|
|
all_flows = [direct_flow(index) for index in range(10_001)]
|
|
for count in (9_999, 10_000, 10_001):
|
|
with self.subTest(collection="flows", count=count):
|
|
inventory = {
|
|
"schema_version": 1,
|
|
"product": "crank-community",
|
|
"flows": all_flows[:count],
|
|
}
|
|
diagnostics, _ = module.validate_inventory(self.root, inventory, ["FR-46"])
|
|
has_limit_error = any(
|
|
item.code == "LIMIT_EXCEEDED" and item.pointer == "/flows"
|
|
for item in diagnostics.ordered()
|
|
)
|
|
self.assertEqual(has_limit_error, count > 10_000)
|
|
|
|
for count in (63, 64, 65):
|
|
with self.subTest(collection="requirements", count=count):
|
|
flow = self.valid_flow(f"requirements-{count}", "api", "FR-1")
|
|
flow["requirements"] = [f"FR-{index}" for index in range(1, count + 1)]
|
|
self.write_inventory([flow])
|
|
result = self.run_validator(["FR-1"])
|
|
self.assertEqual(result.returncode == 0, count <= 64, result.stderr)
|
|
|
|
evidence_paths = [
|
|
self.write_evidence(f"evidence/bounds/path-{index:02d}.txt")
|
|
for index in range(65)
|
|
]
|
|
for count in (63, 64, 65):
|
|
with self.subTest(collection="evidence", count=count):
|
|
flow = self.valid_flow(f"evidence-{count}", "api", "FR-46")
|
|
flow["evidence"] = {
|
|
"automated": evidence_paths[:count],
|
|
"manual": [manual],
|
|
}
|
|
self.write_inventory([flow])
|
|
result = self.run_validator(["FR-46"])
|
|
self.assertEqual(result.returncode == 0, count <= 64, result.stderr)
|
|
|
|
for count in (15, 16, 17):
|
|
with self.subTest(collection="capability-bound", count=count):
|
|
diagnostics = module.validate_string_list(
|
|
[f"capability-{index}" for index in range(count)],
|
|
"/capabilities",
|
|
16,
|
|
128,
|
|
)
|
|
has_limit_error = any(item.code == "LIMIT_EXCEEDED" for item in diagnostics)
|
|
self.assertEqual(has_limit_error, count > 16)
|
|
|
|
def test_evidence_path_limit_covers_below_at_and_above_boundary(self) -> None:
|
|
def path_of_length(length: int) -> str:
|
|
segment_count = (length + 199) // 201
|
|
character_count = length - (segment_count - 1)
|
|
lengths: list[int] = []
|
|
remaining = character_count
|
|
for index in range(segment_count):
|
|
slots_left = segment_count - index - 1
|
|
current = min(200, remaining - slots_left)
|
|
lengths.append(current)
|
|
remaining -= current
|
|
path = "/".join("p" * size for size in lengths)
|
|
self.assertEqual(len(path), length)
|
|
return path
|
|
|
|
manual = self.write_evidence("evidence/manual/path-limit.md")
|
|
for length in (1_023, 1_024, 1_025):
|
|
with self.subTest(length=length):
|
|
evidence_path = path_of_length(length)
|
|
if length <= 1_024:
|
|
self.write_evidence(evidence_path)
|
|
flow = self.valid_flow(f"path-limit-{length}", "api", "FR-46")
|
|
flow["evidence"] = {
|
|
"automated": [evidence_path],
|
|
"manual": [manual],
|
|
}
|
|
self.write_inventory([flow])
|
|
result = self.run_validator(["FR-46"])
|
|
self.assertEqual(result.returncode == 0, length <= 1_024, result.stderr)
|
|
|
|
def test_semantic_validator_enforces_schema_shape_uniqueness_and_fr_format(self) -> None:
|
|
cases: list[tuple[str, object, str]] = []
|
|
|
|
unknown_top = {
|
|
"schema_version": 1,
|
|
"product": "crank-community",
|
|
"flows": [self.valid_flow("unknown-top", "api", "FR-46")],
|
|
"unexpected": True,
|
|
}
|
|
cases.append(("unknown top-level field", unknown_top, "UNKNOWN_FIELD"))
|
|
|
|
unknown_flow = self.valid_flow("unknown-flow", "api", "FR-46")
|
|
unknown_flow["unexpected"] = True
|
|
cases.append(
|
|
(
|
|
"unknown flow field",
|
|
{"schema_version": 1, "product": "crank-community", "flows": [unknown_flow]},
|
|
"UNKNOWN_FIELD",
|
|
)
|
|
)
|
|
|
|
duplicate_requirement = self.valid_flow("duplicate-fr", "api", "FR-46")
|
|
duplicate_requirement["requirements"] = ["FR-46", "FR-46"]
|
|
cases.append(
|
|
(
|
|
"duplicate requirement",
|
|
{
|
|
"schema_version": 1,
|
|
"product": "crank-community",
|
|
"flows": [duplicate_requirement],
|
|
},
|
|
"DUPLICATE_LIST_ITEM",
|
|
)
|
|
)
|
|
|
|
malformed_requirement = self.valid_flow("malformed-fr", "api", "FR-46")
|
|
malformed_requirement["requirements"] = ["FR-0"]
|
|
cases.append(
|
|
(
|
|
"malformed requirement",
|
|
{
|
|
"schema_version": 1,
|
|
"product": "crank-community",
|
|
"flows": [malformed_requirement],
|
|
},
|
|
"INVALID_FORMAT",
|
|
)
|
|
)
|
|
|
|
unknown_evidence = self.valid_flow("unknown-evidence", "api", "FR-46")
|
|
evidence = unknown_evidence["evidence"]
|
|
self.assertIsInstance(evidence, dict)
|
|
evidence["unexpected"] = []
|
|
cases.append(
|
|
(
|
|
"unknown evidence field",
|
|
{
|
|
"schema_version": 1,
|
|
"product": "crank-community",
|
|
"flows": [unknown_evidence],
|
|
},
|
|
"UNKNOWN_FIELD",
|
|
)
|
|
)
|
|
|
|
for label, payload, expected_code in cases:
|
|
with self.subTest(label=label):
|
|
self.inventory_path.write_text(
|
|
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
result = self.run_validator(["FR-46"])
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn(f"error[{expected_code}]", result.stderr)
|
|
|
|
def test_accepts_planned_modern_mcp_and_load_run_capabilities(self) -> None:
|
|
capabilities = ["resources", "prompts", "tasks", "load_runs"]
|
|
flows = [
|
|
self.valid_flow(
|
|
f"planned-{capability.replace('_', '-')}",
|
|
"mcp" if capability != "load_runs" else "ui",
|
|
f"FR-{index}",
|
|
"planned",
|
|
[capability],
|
|
)
|
|
for index, capability in enumerate(capabilities, start=47)
|
|
]
|
|
self.write_inventory(flows)
|
|
|
|
result = self.run_validator([f"FR-{index}" for index in range(47, 51)])
|
|
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
for capability in capabilities:
|
|
self.assertNotIn(f"error[FORBIDDEN_COMMUNITY_CAPABILITY]: {capability}", result.stderr)
|
|
|
|
def test_rejects_forbidden_community_capabilities_despite_allow_text(self) -> None:
|
|
forbidden = [
|
|
"multi_workspace", # community-scope: allow=multi-workspace
|
|
"enterprise_rbac", # community-scope: allow=enterprise
|
|
"sso", # community-scope: allow=sso
|
|
"non_rest_upstream", # community-scope: allow=non-rest-upstream
|
|
"arbitrary_distributed_load_targets", # community-scope: allow=distributed-load-targets
|
|
]
|
|
|
|
for index, capability in enumerate(forbidden, start=1):
|
|
with self.subTest(capability=capability):
|
|
flow = self.valid_flow(
|
|
f"forbidden-{index}",
|
|
"api",
|
|
"FR-46",
|
|
"planned",
|
|
[capability],
|
|
)
|
|
flow["notes"] = f"community-scope: allow={capability}"
|
|
self.write_inventory([flow])
|
|
result = self.run_validator(["FR-46"])
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("error[FORBIDDEN_COMMUNITY_CAPABILITY]", result.stderr)
|
|
self.assertIn(capability, result.stderr)
|
|
|
|
def test_scope_scanner_receives_inventory_files_explicitly_and_cannot_be_bypassed(self) -> None:
|
|
clean_inventory = self.valid_flow(
|
|
"planned-resources",
|
|
"mcp",
|
|
"FR-47",
|
|
"planned",
|
|
["resources"],
|
|
)
|
|
self.write_inventory([clean_inventory])
|
|
explicit_files = [
|
|
str(self.inventory_path.relative_to(self.root)),
|
|
"docs/new-untracked-scope-note.md",
|
|
]
|
|
note = self.root / explicit_files[1]
|
|
note.write_text("Community Resources remain planned.\n", encoding="utf-8")
|
|
|
|
clean = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(COMMUNITY_CHECKER),
|
|
"--root",
|
|
str(self.root),
|
|
"--files",
|
|
*explicit_files,
|
|
],
|
|
check=False,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
self.assertEqual(clean.returncode, 0, clean.stderr)
|
|
self.assertIn("2 files scanned", clean.stdout)
|
|
|
|
note.write_text(
|
|
"community-scope: allow=enterprise\n"
|
|
"enterprise_rbac remains outside Community scope.\n", # community-scope: allow=enterprise
|
|
encoding="utf-8",
|
|
)
|
|
rejected = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(COMMUNITY_CHECKER),
|
|
"--root",
|
|
str(self.root),
|
|
"--files",
|
|
*explicit_files,
|
|
],
|
|
check=False,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
self.assertNotEqual(rejected.returncode, 0)
|
|
self.assertIn("docs/new-untracked-scope-note.md:2", rejected.stderr)
|
|
self.assertIn("forbidden marker", rejected.stderr)
|
|
self.assertIn("enterprise", rejected.stderr) # community-scope: allow=enterprise
|
|
|
|
def test_canonical_inventory_seed_does_not_promote_planned_work(self) -> None:
|
|
for path in [
|
|
CANONICAL_INVENTORY,
|
|
CANONICAL_SCHEMA,
|
|
]:
|
|
self.assertTrue(path.is_file(), f"Missing canonical artifact: {path}")
|
|
|
|
inventory = json.loads(CANONICAL_INVENTORY.read_text(encoding="utf-8"))
|
|
schema = json.loads(CANONICAL_SCHEMA.read_text(encoding="utf-8"))
|
|
flow_schema = schema["$defs"]["flow"]
|
|
evidence_paths = schema["$defs"]["evidencePaths"]
|
|
self.assertEqual(
|
|
schema["$schema"],
|
|
"https://json-schema.org/draft/2020-12/schema",
|
|
)
|
|
self.assertEqual(schema["properties"]["flows"]["maxItems"], 10_000)
|
|
self.assertEqual(
|
|
set(flow_schema["properties"]["type"]["enum"]),
|
|
{"ui", "api", "mcp"},
|
|
)
|
|
self.assertEqual(
|
|
set(flow_schema["properties"]["status"]["enum"]),
|
|
{"implemented", "planned", "gap", "blocked"},
|
|
)
|
|
self.assertEqual(evidence_paths["minItems"], 1)
|
|
self.assertEqual(evidence_paths["maxItems"], 64)
|
|
target_capabilities = {"resources", "prompts", "tasks", "load_runs"}
|
|
matching_flows = [
|
|
flow
|
|
for flow in inventory["flows"]
|
|
if target_capabilities.intersection(flow.get("capabilities", []))
|
|
]
|
|
observed_capabilities = {
|
|
capability
|
|
for flow in matching_flows
|
|
for capability in flow.get("capabilities", [])
|
|
if capability in target_capabilities
|
|
}
|
|
|
|
self.assertEqual(observed_capabilities, target_capabilities)
|
|
self.assertTrue(matching_flows)
|
|
self.assertTrue(
|
|
all(flow["status"] != "implemented" for flow in matching_flows),
|
|
"Story 1.1 must not claim target capabilities are implemented.",
|
|
)
|
|
|
|
def test_canonical_inventory_covers_all_requirements_and_required_surfaces(self) -> None:
|
|
inventory = json.loads(CANONICAL_INVENTORY.read_text(encoding="utf-8"))
|
|
required_surfaces = json.loads(REQUIRED_SURFACES.read_text(encoding="utf-8"))
|
|
flows = {flow["id"]: flow for flow in inventory["flows"]}
|
|
observed_requirements = {
|
|
requirement for flow in flows.values() for requirement in flow["requirements"]
|
|
}
|
|
|
|
self.assertEqual(
|
|
{f"FR-{number}" for number in range(1, 55)} - observed_requirements,
|
|
set(),
|
|
)
|
|
self.assertEqual(
|
|
set(required_surfaces["required_flow_ids"]) - set(flows),
|
|
set(),
|
|
)
|
|
for flow_id in required_surfaces["required_flow_ids"]:
|
|
flow = flows[flow_id]
|
|
self.assertIn(flow["status"], {"implemented", "blocked"})
|
|
for evidence_kind in ("automated", "manual"):
|
|
for evidence_path in flow["evidence"][evidence_kind]:
|
|
resolved = ROOT / evidence_path
|
|
self.assertTrue(resolved.is_file(), evidence_path)
|
|
self.assertFalse(resolved.is_symlink(), evidence_path)
|
|
|
|
def test_tracked_documentation_describes_planned_scope(self) -> None:
|
|
for documentation in [
|
|
COMMUNITY_README,
|
|
DOCS_INDEX,
|
|
COMMUNITY_INTRO,
|
|
COMMUNITY_README_EN,
|
|
]:
|
|
text = documentation.read_text(encoding="utf-8")
|
|
self.assertIn("docs/capability-inventory.json", text)
|
|
self.assertRegex(text.lower(), r"\b(planned|gap|blocked)\b")
|
|
|
|
def test_no_story_acceptance_scenario_remains_skipped(self) -> None:
|
|
for name, method in vars(type(self)).items():
|
|
if not name.startswith("test_") or name == self._testMethodName:
|
|
continue
|
|
self.assertFalse(
|
|
getattr(method, "__unittest_skip__", False),
|
|
f"Story acceptance scenario remains skipped: {name}",
|
|
)
|
|
|
|
def test_canonical_validator_is_a_ci_and_release_prerequisite(self) -> None:
|
|
justfile = JUSTFILE.read_text(encoding="utf-8")
|
|
ci = CI_WORKFLOW.read_text(encoding="utf-8")
|
|
release = RELEASE_WORKFLOW.read_text(encoding="utf-8")
|
|
command = "scripts/validate-capability-inventory.py"
|
|
|
|
self.assertIn("capability-inventory-check:", justfile)
|
|
self.assertIn("just capability-inventory-check", justfile)
|
|
self.assertLess(ci.index("Run tooling unit tests"), ci.index(command))
|
|
self.assertLess(ci.index(command), ci.index("Check Community scope"))
|
|
self.assertIn("python3 --version", release)
|
|
self.assertLess(release.index("Run tooling unit tests"), release.index(command))
|
|
self.assertLess(release.index(command), release.index("Check Community scope"))
|
|
self.assertLess(release.index("Check Community scope"), release.index("Build release binaries"))
|
|
|
|
|
|
# --- Story 3.1 ATDD (red): the approved two-revision MCP window -------------------------------
|
|
# Oracle: tests/fixtures/mcp-revisions/* (owner-approved 2026-10-11 contract, source-backed, not a
|
|
# snapshot of current output). Behaviour is observed on the real validator
|
|
# (scripts/validate-capability-inventory.py) and the real schema copy.
|
|
# The protocol vocabulary here (supported/not_applicable/experimental/deferred) is deliberately
|
|
# separate from the unchanged flow vocabulary (implemented/planned/gap/blocked).
|
|
|
|
PROTOCOL_STATUSES = {"supported", "not_applicable", "experimental", "deferred"}
|
|
|
|
def load_revision_fixture(self, name: str) -> dict[str, object]:
|
|
return json.loads((MCP_REVISION_FIXTURES / name).read_text(encoding="utf-8"))
|
|
|
|
def write_revision_inventory(self, name: str) -> tuple[dict[str, object], dict[str, object]]:
|
|
payload = self.load_revision_fixture(name)
|
|
expected = payload.pop("expected", {})
|
|
for flow in payload.get("flows", []): # type: ignore[union-attr]
|
|
for kind in ("automated", "manual"):
|
|
for relative in flow.get("evidence", {}).get(kind, []):
|
|
self.write_evidence(relative)
|
|
self.inventory_path.write_text(
|
|
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
return payload, expected # type: ignore[return-value]
|
|
|
|
def required_surfaces(self) -> set[str]:
|
|
oracle = self.load_revision_fixture("source-backed-capabilities.json")
|
|
return {
|
|
item["surface"]
|
|
for item in oracle["items"] # type: ignore[index]
|
|
if item["role"] == "server"
|
|
}
|
|
|
|
def test_at_3_1_01_full_reviewed_candidate_passes_the_validity_gate(self) -> None:
|
|
"""Target oracle: the FULL reviewed candidate is an accepted v2 inventory (Gate A, exit 0).
|
|
|
|
This is a genuine behavioural RED on the current revision: the validator has no v2 `protocol`
|
|
block, so it answers `schema_version 1 is required` / `UNKNOWN_FIELD`. The assertions below
|
|
describe the agreed target behaviour and are deliberately NOT relaxed to the current one.
|
|
"""
|
|
self.write_candidate_inventory()
|
|
|
|
result = self.run_validator(["FR-13"])
|
|
|
|
combined = result.stdout + result.stderr
|
|
self.assertEqual(
|
|
result.returncode,
|
|
0,
|
|
"the complete reviewed v2 candidate must pass the validity gate, observed: " + combined,
|
|
)
|
|
# The agreed validity output is the existing one-line summary plus the protocol counts.
|
|
self.assertIn("Capability inventory validation passed", combined)
|
|
self.assertRegex(combined, r"\bprotocol_items=50\b")
|
|
self.assertRegex(combined, r"\brevisions=2\b")
|
|
|
|
def test_at_3_1_01_scoped_pre_atdd_candidate_stays_a_scoped_input(self) -> None:
|
|
"""Preserved pre-ATDD input: the scoped window candidate is NOT promoted to full validity.
|
|
|
|
Pure oracle-input provenance - no validator run, so this is not labelled validator GREEN. The
|
|
file keeps its own recorded verdict (`counts_as_exhaustive: false`, completeness BLOCKED) and
|
|
stays byte-identical (checked by the ATDD snapshot verifier); the reviewed candidate above is
|
|
the completeness basis.
|
|
"""
|
|
payload, expected = self.write_revision_inventory("inventory-two-revision-window.json")
|
|
protocol: Any = payload["protocol"]
|
|
cells = protocol["capabilities"]
|
|
|
|
self.assertEqual(payload["schema_version"], 2)
|
|
self.assertEqual(
|
|
[entry["revision"] for entry in protocol["revisions"]],
|
|
["2026-07-28", "2025-11-25"],
|
|
)
|
|
self.assertEqual(
|
|
set(entry["revision"] for entry in protocol["revisions"])
|
|
& set(protocol["withdrawn"]),
|
|
set(),
|
|
)
|
|
self.assertIn("scope_note", payload)
|
|
self.assertEqual(expected["gate"], "validity-shape")
|
|
self.assertEqual(expected["verdict"], "scoped-candidate-not-full-validity")
|
|
self.assertEqual(expected["completeness"], "blocked-scoped-subset")
|
|
self.assertEqual({cell["surface"] for cell in cells}, self.required_surfaces())
|
|
|
|
def test_at_3_1_01_third_revision_outside_the_approved_window_is_rejected_by_name(self) -> None:
|
|
"""Bounded one-factor mutation OF THE REVIEWED CANDIDATE: one extra ledger revision entry."""
|
|
payload = self.write_candidate_inventory("inventory-mutation-third-revision.json")
|
|
revisions = [entry["revision"] for entry in payload["protocol"]["revisions"]] # type: ignore[index]
|
|
self.assertEqual(revisions[:2], ["2026-07-28", "2025-11-25"])
|
|
self.assertEqual(revisions[2:], ["2025-03-26"], "only one revision entry may be added")
|
|
|
|
result = self.run_validator(["FR-13"])
|
|
|
|
combined = result.stdout + result.stderr
|
|
self.assertNotEqual(result.returncode, 0, result.stdout)
|
|
self.assertIn("2025-03-26", combined)
|
|
|
|
def test_at_3_1_03_reviewed_candidate_omission_is_rejected_by_the_triple(self) -> None:
|
|
"""Bounded one-factor mutation OF THE REVIEWED CANDIDATE: exactly one reviewed row dropped."""
|
|
payload = self.write_candidate_inventory("inventory-mutation-omission.json")
|
|
observed = {
|
|
(cell["role"], cell["kind"], cell["surface"])
|
|
for cell in payload["protocol"]["capabilities"] # type: ignore[index]
|
|
}
|
|
self.assertEqual(len(observed), 49, "only one reviewed identity may be dropped")
|
|
self.assertNotIn(("client", "method", "prompts/get"), observed)
|
|
|
|
result = self.run_validator(["FR-13"])
|
|
|
|
combined = result.stdout + result.stderr
|
|
self.assertNotEqual(result.returncode, 0, result.stdout)
|
|
lines = [line for line in combined.splitlines() if "OMISSION" in line]
|
|
self.assertTrue(
|
|
any("prompts/get" in line for line in lines),
|
|
f"the omitted triple must be printed by name: {combined}",
|
|
)
|
|
self.assertTrue(
|
|
any("client" in line and "method" in line for line in lines),
|
|
f"the full (role, kind, surface) triple must be printed: {combined}",
|
|
)
|
|
|
|
def test_at_3_1_03_undocumented_surface_is_rejected_by_the_triple(self) -> None:
|
|
"""Bounded one-factor mutation OF THE REVIEWED CANDIDATE: one extra undocumented triple."""
|
|
payload = self.write_candidate_inventory("inventory-mutation-undocumented-surface.json")
|
|
observed = {
|
|
(cell["role"], cell["kind"], cell["surface"])
|
|
for cell in payload["protocol"]["capabilities"] # type: ignore[index]
|
|
}
|
|
self.assertEqual(len(observed), 51, "only one extra identity may be added")
|
|
self.assertIn(("server", "method", "resources/read"), observed)
|
|
|
|
result = self.run_validator(["FR-13"])
|
|
|
|
combined = result.stdout + result.stderr
|
|
self.assertNotEqual(result.returncode, 0, result.stdout)
|
|
lines = [line for line in combined.splitlines() if "UNDOCUMENTED_SURFACE" in line]
|
|
self.assertTrue(
|
|
any("resources/read" in line for line in lines),
|
|
f"the undocumented triple must be printed by name: {combined}",
|
|
)
|
|
self.assertTrue(
|
|
any("server" in line and "method" in line for line in lines),
|
|
f"the full (role, kind, surface) triple must be printed: {combined}",
|
|
)
|
|
|
|
def test_at_3_1_01_revision_missing_only_the_digest_is_rejected_by_that_field(self) -> None:
|
|
"""One-factor provenance negative: only `provenance.sha256` of the modern revision is gone."""
|
|
payload = self.write_candidate_inventory("inventory-mutation-missing-digest.json")
|
|
modern = next(
|
|
entry
|
|
for entry in payload["protocol"]["revisions"] # type: ignore[index]
|
|
if entry["revision"] == "2026-07-28"
|
|
)
|
|
self.assertEqual(sorted(modern["provenance"]), ["captured_on", "locator"])
|
|
|
|
result = self.run_validator(["FR-13"])
|
|
|
|
combined = result.stdout + result.stderr
|
|
self.assertNotEqual(result.returncode, 0, result.stdout)
|
|
diagnostics = [line for line in combined.splitlines() if line.startswith("error[")]
|
|
digest_lines = [line for line in diagnostics if "sha256" in line]
|
|
self.assertTrue(
|
|
any("2026-07-28" in line for line in digest_lines),
|
|
f"the missing digest must be reported for the revision that lost it: {combined}",
|
|
)
|
|
self.assertFalse(
|
|
any("captured_on" in line for line in diagnostics),
|
|
f"the date is still present and must not be reported missing: {combined}",
|
|
)
|
|
|
|
def test_at_3_1_01_revision_missing_only_the_date_is_rejected_by_that_field(self) -> None:
|
|
"""One-factor provenance negative: only `provenance.captured_on` of the modern revision is gone."""
|
|
payload = self.write_candidate_inventory("inventory-mutation-missing-date.json")
|
|
modern = next(
|
|
entry
|
|
for entry in payload["protocol"]["revisions"] # type: ignore[index]
|
|
if entry["revision"] == "2026-07-28"
|
|
)
|
|
self.assertEqual(sorted(modern["provenance"]), ["locator", "sha256"])
|
|
|
|
result = self.run_validator(["FR-13"])
|
|
|
|
combined = result.stdout + result.stderr
|
|
self.assertNotEqual(result.returncode, 0, result.stdout)
|
|
diagnostics = [line for line in combined.splitlines() if line.startswith("error[")]
|
|
date_lines = [line for line in diagnostics if "captured_on" in line]
|
|
self.assertTrue(
|
|
any("2026-07-28" in line for line in date_lines),
|
|
f"the missing capture date must be reported for the revision that lost it: {combined}",
|
|
)
|
|
self.assertFalse(
|
|
any("sha256" in line for line in diagnostics),
|
|
f"the digest is still present and must not be reported missing: {combined}",
|
|
)
|
|
|
|
def test_at_3_1_03_ordered_validity_first_gate_table_is_exercised_when_mode_exists(self) -> None:
|
|
"""The agreed validity-first gate table (A0 / B3 / B0 / invalid1) on the real CLI.
|
|
|
|
While the CLI exposes no `--mode`, this stays an honest interface gap (skip, never a
|
|
synthetic red). As soon as `--mode` exists the same test body exercises every agreed row, so
|
|
a wrong Gate A/B implementation fails here instead of being skipped.
|
|
"""
|
|
probe = subprocess.run(
|
|
[sys.executable, str(VALIDATOR), "--help"],
|
|
check=False,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
help_text = probe.stdout + probe.stderr
|
|
self.assertEqual(probe.returncode, 0, help_text)
|
|
if "--mode" not in help_text:
|
|
self.skipTest(
|
|
"interface gap, not a behavioural red: the validator CLI exposes no --mode yet, so "
|
|
"the ordered validity/feature-complete gate table cannot be exercised on this "
|
|
"revision"
|
|
)
|
|
|
|
self.install_canonical_schema()
|
|
|
|
# invalid1 - structurally invalid input: `invalid` in BOTH modes, step-0 diagnostics, and no
|
|
# feature-complete evaluation at all (accepted spec `remaining-ac-spec.md:392`).
|
|
self.inventory_path.write_text("{not-json", encoding="utf-8")
|
|
for mode in ("validity", "feature-complete"):
|
|
with self.subTest(row="structural-invalid", mode=mode):
|
|
result = self.run_validator(["FR-13"], install_schema=False, mode=mode)
|
|
combined = result.stdout + result.stderr
|
|
self.assertEqual(result.returncode, 1, combined)
|
|
self.assertIn("status=invalid", combined)
|
|
self.assertIn("error[", combined)
|
|
self.assertNotIn("status=blocked", combined)
|
|
self.assertNotIn("status=complete", combined)
|
|
self.assertNotRegex(combined, r"blockers=[1-9]")
|
|
|
|
# invalid1 - semantic validity diagnostics (only the modern `provenance.sha256` is gone):
|
|
# `invalid` in BOTH modes, the targeted missing-digest diagnostic, no feature-complete output
|
|
# (accepted spec `remaining-ac-spec.md:393`).
|
|
self.write_candidate_inventory("inventory-mutation-missing-digest.json")
|
|
for mode in ("validity", "feature-complete"):
|
|
with self.subTest(row="semantic-invalid", mode=mode):
|
|
result = self.run_validator(["FR-13"], mode=mode)
|
|
combined = result.stdout + result.stderr
|
|
self.assertEqual(result.returncode, 1, combined)
|
|
self.assertIn("status=invalid", combined)
|
|
diagnostics = [line for line in combined.splitlines() if line.startswith("error[")]
|
|
self.assertTrue(
|
|
any("2026-07-28" in line and "sha256" in line for line in diagnostics),
|
|
f"the missing digest must be named for the revision that lost it: {combined}",
|
|
)
|
|
self.assertNotIn("status=blocked", combined)
|
|
self.assertNotIn("status=complete", combined)
|
|
self.assertNotRegex(combined, r"blockers=[1-9]")
|
|
|
|
# invalid1 - the rejected false-not-applicable variant: applicable scheduled work renamed to
|
|
# `not_applicable` must stay `invalid` in BOTH modes, never `complete` with 0 blockers.
|
|
self.write_candidate_inventory("inventory-mutation-false-not-applicable.json")
|
|
for mode in ("validity", "feature-complete"):
|
|
with self.subTest(row="false-not-applicable", mode=mode):
|
|
result = self.run_validator(["FR-13"], mode=mode)
|
|
combined = result.stdout + result.stderr
|
|
self.assertEqual(result.returncode, 1, combined)
|
|
self.assertIn("status=invalid", combined)
|
|
self.assertNotIn("status=blocked", combined)
|
|
self.assertNotIn("status=complete", combined)
|
|
self.assertNotRegex(combined, r"blockers=[1-9]")
|
|
|
|
# A0 / B3 - a truthful candidate passes validity and is blocked by the feature gate.
|
|
self.write_candidate_inventory()
|
|
validity = self.run_validator(["FR-13"], mode="validity")
|
|
self.assertEqual(validity.returncode, 0, validity.stdout + validity.stderr)
|
|
self.assertIn("Capability inventory validation passed", validity.stdout + validity.stderr)
|
|
feature_complete = self.run_validator(["FR-13"], mode="feature-complete")
|
|
self.assertEqual(
|
|
feature_complete.returncode,
|
|
3,
|
|
feature_complete.stdout + feature_complete.stderr,
|
|
)
|
|
self.assertIn("status=blocked", feature_complete.stdout + feature_complete.stderr)
|
|
self.assertRegex(
|
|
feature_complete.stdout + feature_complete.stderr,
|
|
r"blockers=[1-9][0-9]*",
|
|
)
|
|
|
|
# B0 - the explicitly SYNTHETIC no-blocker input (hypothetical completed state, marked with
|
|
# its own owner) is complete in the feature gate.
|
|
self.write_candidate_inventory("inventory-synthetic-no-blocker.json")
|
|
validity = self.run_validator(["FR-13"], mode="validity")
|
|
self.assertEqual(validity.returncode, 0, validity.stdout + validity.stderr)
|
|
self.assertIn("Capability inventory validation passed", validity.stdout + validity.stderr)
|
|
feature_complete = self.run_validator(["FR-13"], mode="feature-complete")
|
|
self.assertEqual(
|
|
feature_complete.returncode,
|
|
0,
|
|
feature_complete.stdout + feature_complete.stderr,
|
|
)
|
|
self.assertIn("status=complete", feature_complete.stdout + feature_complete.stderr)
|
|
self.assertRegex(
|
|
feature_complete.stdout + feature_complete.stderr,
|
|
r"blockers=0\b",
|
|
)
|
|
|
|
# --- Story 3.1 remaining-foundation ATDD additions ------------------------------------------
|
|
# Oracle inputs: the reviewed v2 candidate (`inventory-reviewed-v2-candidate.json`), its bounded
|
|
# one-factor mutations, and the independent per-era frozen projections. The reviewed identity is
|
|
# the triple `(role, kind, surface)`; `role` is one enum {server, client, both} with a
|
|
# kind-specific meaning (declarer for capability/extension, sender for method/notification).
|
|
# The target validator behaviour (v2 protocol block, Gate A/B split, named diagnostics) does not
|
|
# exist on this revision, so the assertions below are the contract the implementation stage must
|
|
# satisfy. Anything that can only fail because a CLI flag or a module is absent is recorded as an
|
|
# honest API gap, never as a behavioural red and never as a pass.
|
|
|
|
REVIEWED_IDENTITY_TRIPLES = {
|
|
# server capabilities (surface-inventory.md section 3)
|
|
("server", "capability", "experimental"),
|
|
("server", "capability", "logging"),
|
|
("server", "capability", "completions"),
|
|
("server", "capability", "prompts"),
|
|
("server", "capability", "resources"),
|
|
("server", "capability", "tools"),
|
|
("server", "capability", "extensions"),
|
|
("server", "capability", "tasks"),
|
|
# client capabilities (section 4)
|
|
("client", "capability", "experimental"),
|
|
("client", "capability", "roots"),
|
|
("client", "capability", "sampling"),
|
|
("client", "capability", "elicitation"),
|
|
("client", "capability", "extensions"),
|
|
("client", "capability", "tasks"),
|
|
# methods (section 5); role = the sending party
|
|
("client", "method", "tools/list"),
|
|
("client", "method", "tools/call"),
|
|
("client", "method", "resources/list"),
|
|
("client", "method", "resources/read"),
|
|
("client", "method", "resources/templates/list"),
|
|
("client", "method", "resources/subscribe"),
|
|
("client", "method", "resources/unsubscribe"),
|
|
("client", "method", "prompts/list"),
|
|
("client", "method", "prompts/get"),
|
|
("client", "method", "completion/complete"),
|
|
("client", "method", "logging/setLevel"),
|
|
("client", "method", "initialize"),
|
|
("both", "method", "ping"),
|
|
("client", "method", "server/discover"),
|
|
("client", "method", "subscriptions/listen"),
|
|
("server", "method", "roots/list"),
|
|
("server", "method", "sampling/createMessage"),
|
|
("server", "method", "elicitation/create"),
|
|
("both", "method", "tasks/get"),
|
|
("both", "method", "tasks/list"),
|
|
("both", "method", "tasks/result"),
|
|
("both", "method", "tasks/cancel"),
|
|
# notifications (section 6)
|
|
("server", "notification", "notifications/tools/list_changed"),
|
|
("server", "notification", "notifications/resources/list_changed"),
|
|
("server", "notification", "notifications/prompts/list_changed"),
|
|
("server", "notification", "notifications/resources/updated"),
|
|
("server", "notification", "notifications/progress"),
|
|
("server", "notification", "notifications/message"),
|
|
("server", "notification", "notifications/subscriptions/acknowledged"),
|
|
("both", "notification", "notifications/cancelled"),
|
|
("client", "notification", "notifications/initialized"),
|
|
("client", "notification", "notifications/roots/list_changed"),
|
|
("both", "notification", "notifications/tasks/status"),
|
|
("server", "notification", "notifications/elicitation/complete"),
|
|
# extensions (section 7)
|
|
("server", "extension", "io.modelcontextprotocol/tasks"),
|
|
("client", "extension", "io.modelcontextprotocol/oauth-client-credentials"),
|
|
}
|
|
|
|
REVIEWED_ROLES = {"server", "client", "both"}
|
|
REVIEWED_KINDS = {"capability", "method", "notification", "extension"}
|
|
REVIEWED_STATUSES = {"supported", "not_applicable", "experimental", "deferred"}
|
|
|
|
# Independent per-era ABSENCE oracle, read off the "absent" cells of the reviewed tables in
|
|
# `build/remaining-ac-spec/surface-inventory.md` sections 3-7 (deliberately NOT taken from the
|
|
# fixture). Rule O2 resolves an era in two steps (`remaining-ac-spec.md:296-300`): a surface whose
|
|
# `revisions[]` does not contain the era is ABSENT for that era - never `not_applicable` - and only
|
|
# then is the shallow `per_revision[era]` merge applied. Applicable per-era completeness is
|
|
# therefore `REVIEWED_IDENTITY_TRIPLES - REVIEWED_ABSENT_<era>`, not the 50-identity default union.
|
|
REVIEWED_ABSENT_MODERN = {
|
|
# sections 3/4: modern expresses Tasks as an extension, so the server `tasks` capability is
|
|
# absent, and the client `tasks` declaration does not exist in modern either
|
|
("server", "capability", "tasks"),
|
|
("client", "capability", "tasks"),
|
|
# section 5: method literals absent from the pinned modern schema
|
|
("client", "method", "resources/subscribe"),
|
|
("client", "method", "resources/unsubscribe"),
|
|
("client", "method", "logging/setLevel"),
|
|
("client", "method", "initialize"),
|
|
("both", "method", "ping"),
|
|
("both", "method", "tasks/get"),
|
|
("both", "method", "tasks/list"),
|
|
("both", "method", "tasks/result"),
|
|
("both", "method", "tasks/cancel"),
|
|
# section 6: notifications absent from the pinned modern schema
|
|
("client", "notification", "notifications/initialized"),
|
|
("client", "notification", "notifications/roots/list_changed"),
|
|
("both", "notification", "notifications/tasks/status"),
|
|
("server", "notification", "notifications/elicitation/complete"),
|
|
}
|
|
REVIEWED_ABSENT_LEGACY = {
|
|
# sections 3/4: neither legacy `ServerCapabilities` nor legacy `ClientCapabilities` has an
|
|
# `extensions` key
|
|
("server", "capability", "extensions"),
|
|
("client", "capability", "extensions"),
|
|
# sections 5/6: modern-only method literals and the subscription acknowledgement
|
|
("client", "method", "server/discover"),
|
|
("client", "method", "subscriptions/listen"),
|
|
("server", "notification", "notifications/subscriptions/acknowledged"),
|
|
# section 7: both extension identifiers are declared in the modern schema only
|
|
("server", "extension", "io.modelcontextprotocol/tasks"),
|
|
("client", "extension", "io.modelcontextprotocol/oauth-client-credentials"),
|
|
}
|
|
|
|
def load_candidate_fixture(self, name: str = "inventory-reviewed-v2-candidate.json") -> dict[str, object]:
|
|
return self.load_revision_fixture(name)
|
|
|
|
def write_candidate_inventory(self, name: str = "inventory-reviewed-v2-candidate.json") -> dict[str, object]:
|
|
"""Install a reviewed candidate/mutation fixture as the validator input."""
|
|
payload = self.load_candidate_fixture(name)
|
|
for flow in payload.get("flows", []): # type: ignore[union-attr]
|
|
for kind in ("automated", "manual"):
|
|
for relative in flow.get("evidence", {}).get(kind, []):
|
|
self.write_evidence(relative)
|
|
self.inventory_path.write_text(
|
|
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
return payload
|
|
|
|
def effective_entry(self, entry: dict[str, object], revision: str) -> dict[str, object]:
|
|
merged = {key: value for key, value in entry.items() if key != "per_revision"}
|
|
overrides = entry.get("per_revision", {}).get(revision, {}) # type: ignore[union-attr]
|
|
merged.update(overrides)
|
|
return merged
|
|
|
|
def test_at_3_1_03_reviewed_candidate_is_identity_complete_and_metadata_stays_separate(self) -> None:
|
|
"""The reviewed completeness basis itself: one judged entry per reviewed row, no more, no less."""
|
|
payload = self.load_candidate_fixture()
|
|
protocol: Any = payload["protocol"]
|
|
|
|
self.assertEqual(payload["schema_version"], 2)
|
|
self.assertEqual(
|
|
[entry["revision"] for entry in protocol["revisions"]],
|
|
["2026-07-28", "2025-11-25"],
|
|
)
|
|
for entry in protocol["revisions"]:
|
|
provenance = entry["provenance"]
|
|
self.assertTrue(provenance["locator"] and provenance["sha256"] and provenance["captured_on"])
|
|
self.assertEqual(protocol["withdrawn"], ["2025-06-18", "2025-03-26"])
|
|
|
|
capabilities = protocol["capabilities"]
|
|
triples = [(cell["role"], cell["kind"], cell["surface"]) for cell in capabilities]
|
|
self.assertEqual(len(triples), len(set(triples)), "identity is the (role, kind, surface) triple")
|
|
self.assertEqual(set(triples), self.REVIEWED_IDENTITY_TRIPLES)
|
|
for cell in capabilities:
|
|
self.assertIn(cell["role"], self.REVIEWED_ROLES)
|
|
self.assertIn(cell["kind"], self.REVIEWED_KINDS)
|
|
self.assertIn(cell["status"], self.REVIEWED_STATUSES)
|
|
self.assertTrue(cell["owner"] and cell["fallback"] and cell["evidence_target"])
|
|
if cell["status"] == "supported":
|
|
self.assertNotIn("reason", cell)
|
|
else:
|
|
self.assertTrue(cell["reason"], cell)
|
|
|
|
# `io.modelcontextprotocol/related-task` is a non-advertisable `_meta` metadata note, not a
|
|
# capability extension: it must stay out of the four-kind ledger and out of the advertisement.
|
|
self.assertFalse(
|
|
any(
|
|
cell["surface"] == "io.modelcontextprotocol/related-task"
|
|
for cell in capabilities
|
|
),
|
|
"related-task must not be recorded as a capability/extension entry",
|
|
)
|
|
|
|
# Per-revision overrides are resolved with a shallow per-field merge (rule O2).
|
|
progress = next(
|
|
cell
|
|
for cell in capabilities
|
|
if cell["surface"] == "notifications/progress" and cell["kind"] == "notification"
|
|
)
|
|
self.assertEqual(self.effective_entry(progress, "2025-11-25")["role"], "both")
|
|
self.assertEqual(self.effective_entry(progress, "2026-07-28")["role"], "server")
|
|
roots = next(
|
|
cell
|
|
for cell in capabilities
|
|
if cell["surface"] == "roots" and cell["kind"] == "capability"
|
|
)
|
|
self.assertEqual(self.effective_entry(roots, "2026-07-28")["subfeatures"], [])
|
|
self.assertEqual(self.effective_entry(roots, "2025-11-25")["subfeatures"], ["listChanged"])
|
|
|
|
# A `both` row satisfies both sender sides but is never advertisable.
|
|
tools = next(cell for cell in capabilities if cell["surface"] == "tools" and cell["role"] == "server")
|
|
self.assertEqual(tools["status"], "supported")
|
|
self.assertEqual(tools["advertised"], {"tools": {"listChanged": False}})
|
|
for cell in capabilities:
|
|
resolved = [self.effective_entry(cell, revision) for revision in ("2026-07-28", "2025-11-25")]
|
|
for effective in resolved:
|
|
if effective["role"] != "server" or effective["kind"] not in {"capability", "extension"}:
|
|
self.assertIsNone(effective["advertised"], cell["surface"])
|
|
if effective["kind"] in {"method", "notification"}:
|
|
self.assertIsNone(effective["advertised"], cell["surface"])
|
|
|
|
def test_at_3_1_03_duplicate_surface_identity_is_rejected_by_the_triple(self) -> None:
|
|
self.write_candidate_inventory("inventory-mutation-duplicate-identity.json")
|
|
|
|
result = self.run_validator(["FR-13"])
|
|
|
|
combined = result.stdout + result.stderr
|
|
self.assertNotEqual(result.returncode, 0, result.stdout)
|
|
self.assertIn("DUPLICATE_SURFACE_IDENTITY", combined)
|
|
self.assertIn("experimental", combined)
|
|
|
|
def test_at_3_1_03_advertised_fragment_on_a_non_advertisable_role_is_rejected(self) -> None:
|
|
self.write_candidate_inventory("inventory-mutation-advertised-on-client-role.json")
|
|
|
|
result = self.run_validator(["FR-13"])
|
|
|
|
combined = result.stdout + result.stderr
|
|
self.assertNotEqual(result.returncode, 0, result.stdout)
|
|
self.assertIn("ADVERTISED_ON_NON_ADVERTISABLE", combined)
|
|
self.assertIn("elicitation", combined)
|
|
|
|
def test_at_3_1_03_supported_server_capability_must_carry_an_advertised_fragment(self) -> None:
|
|
self.write_candidate_inventory("inventory-mutation-supported-server-without-advertised.json")
|
|
|
|
result = self.run_validator(["FR-13"])
|
|
|
|
combined = result.stdout + result.stderr
|
|
self.assertNotEqual(result.returncode, 0, result.stdout)
|
|
self.assertIn("tools", combined)
|
|
self.assertIn("advertised", combined.lower())
|
|
|
|
def test_at_3_1_03_effective_per_era_identities_follow_eligibility_then_o2(self) -> None:
|
|
"""O2 coverage beyond the default 50 identities: effective per-era identity sets.
|
|
|
|
Rule O2 resolves an era in two steps (accepted spec `remaining-ac-spec.md:296-300`): a surface
|
|
whose `revisions[]` does not contain the era is ABSENT for that era - never `not_applicable` -
|
|
and only then is the shallow `per_revision[era]` merge applied. The expectations below are the
|
|
reviewed absence sets read off `surface-inventory.md` sections 3-7, not a copy of the fixture:
|
|
the applicable set of an era is `REVIEWED_IDENTITY_TRIPLES - REVIEWED_ABSENT_<era>` (35 modern,
|
|
43 legacy on this candidate). The 50-identity default union stays a separate ORACLE-INPUT and is
|
|
never reported as per-era completeness.
|
|
"""
|
|
payload = self.load_candidate_fixture()
|
|
protocol: Any = payload["protocol"]
|
|
capabilities: Any = protocol["capabilities"]
|
|
|
|
per_era: dict[str, list[tuple[Any, Any, Any]]] = {}
|
|
progress_reviewed_default = ("server", "notification", "notifications/progress")
|
|
# The one reviewed per-era identity substitution (surface-inventory.md section 2.1/6): the
|
|
# progress sender is `server` in the modern revision and `both` in the legacy revision.
|
|
reviewed_progress_per_era = {
|
|
"2026-07-28": ("server", "notification", "notifications/progress"),
|
|
"2025-11-25": ("both", "notification", "notifications/progress"),
|
|
}
|
|
for era, absent in (
|
|
("2026-07-28", self.REVIEWED_ABSENT_MODERN),
|
|
("2025-11-25", self.REVIEWED_ABSENT_LEGACY),
|
|
):
|
|
with self.subTest(era=era):
|
|
eligible = [cell for cell in capabilities if era in cell["revisions"]]
|
|
triples = [
|
|
(self.effective_entry(cell, era)["role"], cell["kind"], cell["surface"])
|
|
for cell in eligible
|
|
]
|
|
per_era[era] = triples
|
|
expected = (
|
|
self.REVIEWED_IDENTITY_TRIPLES
|
|
- absent
|
|
- {progress_reviewed_default}
|
|
) | {reviewed_progress_per_era[era]}
|
|
|
|
self.assertEqual(
|
|
len(triples),
|
|
len(set(triples)),
|
|
"an override must not create a duplicate EFFECTIVE identity",
|
|
)
|
|
self.assertEqual(
|
|
set(triples),
|
|
expected,
|
|
f"the applicable identities of {era} must match the reviewed tables",
|
|
)
|
|
self.assertEqual(
|
|
sorted(absent),
|
|
sorted(
|
|
(cell["role"], cell["kind"], cell["surface"])
|
|
for cell in capabilities
|
|
if era not in cell["revisions"]
|
|
),
|
|
"every explicitly absent reviewed identity must be ineligible in that era",
|
|
)
|
|
self.assertEqual(
|
|
set(triples) & absent,
|
|
set(),
|
|
"an absent identity can never be part of the applicable set",
|
|
)
|
|
|
|
self.assertEqual(len(per_era["2026-07-28"]), 35)
|
|
self.assertEqual(len(per_era["2025-11-25"]), 43)
|
|
for era, triples in per_era.items():
|
|
self.assertNotEqual(
|
|
set(triples),
|
|
self.REVIEWED_IDENTITY_TRIPLES,
|
|
f"the default 50-identity union is not the applicable set of {era}",
|
|
)
|
|
|
|
# The only identity that resolves to a different ROLE per era is the agreed progress override.
|
|
overridden = set()
|
|
for cell in capabilities:
|
|
if "2026-07-28" in cell["revisions"] and "2025-11-25" in cell["revisions"]:
|
|
roles = {
|
|
era: self.effective_entry(cell, era)["role"]
|
|
for era in ("2026-07-28", "2025-11-25")
|
|
}
|
|
if roles["2026-07-28"] != roles["2025-11-25"]:
|
|
overridden.add((cell["role"], cell["kind"], cell["surface"]))
|
|
self.assertEqual(
|
|
overridden,
|
|
{("server", "notification", "notifications/progress")},
|
|
"the agreed per-era role difference is exactly the progress sender override",
|
|
)
|
|
|
|
progress = next(
|
|
cell
|
|
for cell in capabilities
|
|
if cell["surface"] == "notifications/progress" and cell["kind"] == "notification"
|
|
)
|
|
self.assertEqual(progress["role"], "server")
|
|
self.assertEqual(progress["per_revision"]["2025-11-25"]["role"], "both")
|
|
self.assertEqual(self.effective_entry(progress, "2026-07-28")["role"], "server")
|
|
self.assertEqual(self.effective_entry(progress, "2025-11-25")["role"], "both")
|
|
|
|
# --- B0 / invalid1 inputs of the agreed Gate table (round-2 corrections) ----------------------
|
|
# Both are fixture oracles only: they inspect the input documents and never run the validator, so
|
|
# nothing here is called validator GREEN. The executable Gate rows live in the `--mode` table above
|
|
# and stay an honest interface gap until the CLI exposes `--mode`.
|
|
|
|
def test_at_3_1_03_synthetic_no_blocker_variant_is_a_hypothetical_completed_state(self) -> None:
|
|
"""`inventory-synthetic-no-blocker.json` (B0 row) is synthetic, never a status relabel.
|
|
|
|
Every `deferred` reviewed surface is resolved to the hypothetical completed state
|
|
(`supported`), never to `not_applicable`; advertisable server capability/extension rows carry
|
|
the exact `advertised` fragment their reviewed `subfeatures` declare; and every flipped row is
|
|
marked with the owner `atdd-synthetic-no-blocker-input`. The document therefore contains no
|
|
deferral at all - a no-blocker input under the agreed gate reading and under any stricter one.
|
|
"""
|
|
payload = self.load_candidate_fixture("inventory-synthetic-no-blocker.json")
|
|
protocol: Any = payload["protocol"]
|
|
capabilities: Any = protocol["capabilities"]
|
|
reviewed_payload: dict[str, Any] = self.load_candidate_fixture()
|
|
reviewed: Any = reviewed_payload["protocol"]["capabilities"]
|
|
reviewed_by_identity = {
|
|
(cell["role"], cell["kind"], cell["surface"]): cell for cell in reviewed
|
|
}
|
|
|
|
self.assertEqual(
|
|
[cell["surface"] for cell in capabilities if cell["status"] == "deferred"],
|
|
[],
|
|
"the no-blocker input must not defer any reviewed surface",
|
|
)
|
|
self.assertEqual(
|
|
sorted(cell["surface"] for cell in capabilities if cell["status"] == "not_applicable"),
|
|
sorted(cell["surface"] for cell in reviewed if cell["status"] == "not_applicable"),
|
|
"the reviewed `not_applicable` rows must stay exactly as reviewed",
|
|
)
|
|
|
|
flipped = 0
|
|
for cell in capabilities:
|
|
original = reviewed_by_identity[(cell["role"], cell["kind"], cell["surface"])]
|
|
if original["status"] != "deferred":
|
|
self.assertEqual(
|
|
cell,
|
|
original,
|
|
f"only deferred rows may differ from the reviewed candidate: {cell['surface']}",
|
|
)
|
|
continue
|
|
flipped += 1
|
|
self.assertEqual(cell["status"], "supported", cell["surface"])
|
|
self.assertNotIn("reason", cell, "a hypothetical `supported` row carries no reason")
|
|
self.assertEqual(cell["owner"], "atdd-synthetic-no-blocker-input", cell["surface"])
|
|
self.assertEqual(
|
|
sorted(cell),
|
|
sorted(set(original) - {"reason"}),
|
|
"a flipped row may only change status/reason/advertised/open_subfeatures",
|
|
)
|
|
self.assertEqual(flipped, 26, "every deferred reviewed row must be flipped")
|
|
|
|
for cell in capabilities:
|
|
if cell["owner"].startswith("atdd-synthetic") and cell["role"] == "server" and cell[
|
|
"kind"
|
|
] in {"capability", "extension"}:
|
|
self.assertIsNotNone(
|
|
cell["advertised"],
|
|
f"a supported server declaration must advertise: {cell['surface']}",
|
|
)
|
|
|
|
def test_at_3_1_03_false_not_applicable_input_keeps_the_planned_reasons(self) -> None:
|
|
"""`inventory-mutation-false-not-applicable.json` (invalid1 row) is the REJECTED predecessor.
|
|
|
|
It is the variant that renames every `deferred` status to `not_applicable` while keeping the
|
|
planned/deferred reasons, i.e. it claims that applicable, scheduled work is inapplicable. A
|
|
truthful validity gate must answer `invalid` in BOTH modes and never `complete` / 0 blockers;
|
|
the executable assertion lives in the `--mode` table, which is an honest interface gap until
|
|
`--mode` exists.
|
|
"""
|
|
payload = self.load_candidate_fixture("inventory-mutation-false-not-applicable.json")
|
|
protocol: Any = payload["protocol"]
|
|
capabilities: Any = protocol["capabilities"]
|
|
reviewed_payload: dict[str, Any] = self.load_candidate_fixture()
|
|
reviewed: Any = reviewed_payload["protocol"]["capabilities"]
|
|
reviewed_by_identity = {
|
|
(cell["role"], cell["kind"], cell["surface"]): cell for cell in reviewed
|
|
}
|
|
|
|
relabelled = 0
|
|
falsely_inapplicable_server_capabilities = []
|
|
for cell in capabilities:
|
|
original = reviewed_by_identity[(cell["role"], cell["kind"], cell["surface"])]
|
|
if original["status"] != "deferred":
|
|
self.assertEqual(cell, original, f"only deferred rows may be relabelled: {cell['surface']}")
|
|
continue
|
|
relabelled += 1
|
|
self.assertEqual(cell["status"], "not_applicable", cell["surface"])
|
|
self.assertEqual(
|
|
cell["reason"],
|
|
original["reason"],
|
|
"the relabel keeps the planned/deferred reason - that is the false claim",
|
|
)
|
|
if cell["role"] == "server" and cell["kind"] == "capability":
|
|
falsely_inapplicable_server_capabilities.append(cell["surface"])
|
|
|
|
self.assertEqual(relabelled, 26, "every deferred reviewed row must be relabelled")
|
|
self.assertEqual(
|
|
sorted(falsely_inapplicable_server_capabilities),
|
|
["completions", "logging", "prompts", "tasks"],
|
|
"applicable server capabilities must never be declared inapplicable",
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|