import copy import importlib.util import json import re import shutil import subprocess import sys import tempfile import unittest from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[2] VALIDATOR = ROOT / "scripts" / "validate-capability-inventory.py" CANONICAL_INVENTORY = ROOT / "docs" / "capability-inventory.json" CANONICAL_SCHEMA = ROOT / "docs" / "schemas" / "capability-inventory.schema.json" COMMUNITY_CHECKER = ROOT / "scripts" / "check-community-scope.py" COMMUNITY_README = ROOT / "README.md" DOCS_INDEX = ROOT / "docs" / "README.md" COMMUNITY_INTRO = ROOT / "docs" / "intro.md" COMMUNITY_README_EN = ROOT / "docs" / "en" / "README.md" JUSTFILE = ROOT / "justfile" CI_WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml" RELEASE_WORKFLOW = ROOT / ".gitea" / "workflows" / "release.yml" REQUIRED_SURFACES = ROOT / "docs" / "capability-baseline" / "required-surfaces.json" MCP_REVISION_FIXTURES = ROOT / "tests" / "fixtures" / "mcp-revisions" def load_validator_module(): spec = importlib.util.spec_from_file_location("capability_inventory_validator", VALIDATOR) if spec is None or spec.loader is None: raise RuntimeError("cannot load capability inventory validator") module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module class CapabilityInventoryValidatorTests(unittest.TestCase): def setUp(self) -> None: self._temporary_directory = tempfile.TemporaryDirectory() self.root = Path(self._temporary_directory.name) self.inventory_path = self.root / "docs" / "capability-inventory.json" self.schema_path = ( self.root / "docs" / "schemas" / "capability-inventory.schema.json" ) self.inventory_path.parent.mkdir(parents=True) self.schema_path.parent.mkdir(parents=True) def tearDown(self) -> None: self._temporary_directory.cleanup() def write_evidence(self, relative_path: str, content: str = "verified\n") -> str: path = self.root / relative_path path.parent.mkdir(parents=True, exist_ok=True) path.write_text(content, encoding="utf-8") return relative_path def valid_flow( self, flow_id: str, flow_type: str, requirement: str, status: str = "implemented", capabilities: list[str] | None = None, ) -> dict[str, object]: automated = self.write_evidence( f"evidence/automated/{flow_id}.txt", f"{flow_id}: automated acceptance evidence\n", ) manual = self.write_evidence( f"evidence/manual/{flow_id}.md", f"# {flow_id}\n\nManual verification complete.\n", ) return { "id": flow_id, "type": flow_type, "requirements": [requirement], "user_outcome": f"User completes {flow_id.replace('-', ' ')}.", "owner": "community-release", "status": status, "capabilities": capabilities or ["tools"], "evidence": { "automated": [automated], "manual": [manual], }, } def write_inventory(self, flows: list[dict[str, object]]) -> None: payload = { "schema_version": 1, "product": "crank-community", "flows": flows, } self.inventory_path.write_text( json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) def install_canonical_schema(self) -> None: self.assertTrue( CANONICAL_SCHEMA.is_file(), f"Story 1.1 must create {CANONICAL_SCHEMA.relative_to(ROOT)}", ) shutil.copyfile(CANONICAL_SCHEMA, self.schema_path) def run_validator( self, required_frs: list[str], install_schema: bool = True, mode: str | None = None, ) -> subprocess.CompletedProcess[str]: if install_schema: self.install_canonical_schema() command = [ sys.executable, str(VALIDATOR), "--root", str(self.root), "--inventory", str(self.inventory_path.relative_to(self.root)), "--schema", str(self.schema_path.relative_to(self.root)), ] for requirement in required_frs: command.extend(["--required-fr", requirement]) if mode is not None: command.extend(["--mode", mode]) return subprocess.run( command, check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) def test_accepts_complete_inventory_with_all_flow_types_and_statuses(self) -> None: flows = [ self.valid_flow("ui-create-method", "ui", "FR-1", "implemented"), self.valid_flow("api-create-method", "api", "FR-2", "planned"), self.valid_flow("mcp-tool-call", "mcp", "FR-3", "gap"), self.valid_flow("mcp-resource-read", "mcp", "FR-4", "blocked"), ] self.write_inventory(flows) result = self.run_validator(["FR-1", "FR-2", "FR-3", "FR-4"]) self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("Capability inventory validation passed", result.stdout) self.assertRegex(result.stdout, r"\btotal=4\b") self.assertRegex(result.stdout, r"\bimplemented=1\b") self.assertRegex(result.stdout, r"\bplanned=1\b") self.assertRegex(result.stdout, r"\bgap=1\b") self.assertRegex(result.stdout, r"\bblocked=1\b") def test_counts_only_implemented_flows_as_pass(self) -> None: flows = [ self.valid_flow("implemented-flow", "api", "FR-1", "implemented"), self.valid_flow("planned-flow", "ui", "FR-2", "planned"), self.valid_flow("gap-flow", "mcp", "FR-3", "gap"), self.valid_flow("blocked-flow", "mcp", "FR-4", "blocked"), ] self.write_inventory(flows) result = self.run_validator(["FR-1", "FR-2", "FR-3", "FR-4"]) self.assertEqual(result.returncode, 0, result.stderr) self.assertRegex(result.stdout, r"\bpass=1\b") self.assertNotRegex(result.stdout, r"\bpass=4\b") def test_rejects_duplicate_id_unknown_enum_and_missing_required_fields(self) -> None: base = self.valid_flow("flow-a", "api", "FR-1") cases: list[tuple[str, list[dict[str, object]], str]] = [] duplicate = copy.deepcopy(base) duplicate["user_outcome"] = "A distinct outcome with a duplicate identifier." cases.append(("duplicate ID", [base, duplicate], "DUPLICATE_FLOW_ID")) unknown_status = copy.deepcopy(base) unknown_status["status"] = "done" cases.append(("unknown status", [unknown_status], "UNKNOWN_STATUS")) unknown_type = copy.deepcopy(base) unknown_type["type"] = "worker" cases.append(("unknown type", [unknown_type], "UNKNOWN_FLOW_TYPE")) no_owner = copy.deepcopy(base) no_owner.pop("owner") cases.append(("missing owner", [no_owner], "MISSING_OWNER")) no_outcome = copy.deepcopy(base) no_outcome.pop("user_outcome") cases.append(("missing required field", [no_outcome], "MISSING_REQUIRED_FIELD")) for label, flows, expected_code in cases: with self.subTest(label=label): self.write_inventory(flows) result = self.run_validator(["FR-1"]) self.assertNotEqual(result.returncode, 0) self.assertIn(f"error[{expected_code}]", result.stderr) def test_rejects_missing_mandatory_requirement(self) -> None: self.write_inventory([self.valid_flow("only-fr-one", "api", "FR-1")]) result = self.run_validator(["FR-1", "FR-46"]) self.assertNotEqual(result.returncode, 0) self.assertIn("error[MISSING_REQUIRED_FR]", result.stderr) self.assertIn("FR-46", result.stderr) def test_rejects_missing_non_regular_and_escaping_evidence_links(self) -> None: base = self.valid_flow("evidence-flow", "mcp", "FR-46") outside = self.root.parent / f"{self.root.name}-outside.txt" outside.write_text("outside\n", encoding="utf-8") self.addCleanup(outside.unlink, missing_ok=True) internal_target = self.write_evidence("evidence/automated/internal-target.txt") internal_link = self.root / "evidence" / "automated" / "internal-link.txt" internal_link.symlink_to(self.root / internal_target) outside_link = self.root / "evidence" / "automated" / "outside-link.txt" outside_link.symlink_to(outside) broken_cases = [ "evidence/automated/does-not-exist.txt", "evidence/automated", f"../{outside.name}", str(outside), "evidence/automated/internal-link.txt", "evidence/automated/outside-link.txt", ] for evidence_path in broken_cases: with self.subTest(evidence_path=evidence_path): flow = copy.deepcopy(base) flow["evidence"] = { "automated": [evidence_path], "manual": ["evidence/manual/evidence-flow.md"], } self.write_inventory([flow]) result = self.run_validator(["FR-46"]) self.assertNotEqual(result.returncode, 0) self.assertIn("error[BROKEN_EVIDENCE_LINK]", result.stderr) def test_report_is_deterministic_bounded_sorted_and_redacted(self) -> None: secret_canary = "SECRET_CANARY_sk-live-4hQ9Vw8x" flows = [] for index in range(1_200, 0, -1): flow = self.valid_flow( f"invalid-flow-{index:04d}", "api", f"FR-{index}", ) flow["status"] = "unknown" flow["user_outcome"] = f"{secret_canary}:{index}" flows.append(flow) self.write_inventory(flows) raw_inventory = self.inventory_path.read_text(encoding="utf-8") first = self.run_validator(["FR-46"]) second = self.run_validator(["FR-46"]) self.assertNotEqual(first.returncode, 0) self.assertEqual(first.stderr, second.stderr) self.assertLessEqual(len(first.stderr.encode("utf-8")), 64 * 1024) self.assertNotIn(secret_canary, first.stderr) self.assertNotIn(raw_inventory, first.stderr) diagnostics = re.findall(r"error\[([^]]+)\] ([^:]+):", first.stderr) self.assertTrue(diagnostics) self.assertEqual(diagnostics[:-1], sorted(diagnostics[:-1])) self.assertEqual(diagnostics[-1][0], "TRUNCATED") self.assertNotRegex(first.stderr, r"invalid-flow-\d{4}") def test_rejects_malformed_documents_and_schema_contract_drift(self) -> None: self.write_inventory([self.valid_flow("valid-flow", "api", "FR-46")]) self.inventory_path.write_text("{not-json", encoding="utf-8") malformed_inventory = self.run_validator(["FR-46"]) self.assertNotEqual(malformed_inventory.returncode, 0) self.assertIn("error[INVALID_JSON]", malformed_inventory.stderr) self.write_inventory([self.valid_flow("valid-flow", "api", "FR-46")]) self.schema_path.write_text("{not-json", encoding="utf-8") malformed_schema = self.run_validator(["FR-46"], install_schema=False) self.assertNotEqual(malformed_schema.returncode, 0) self.assertIn("error[INVALID_SCHEMA]", malformed_schema.stderr) self.install_canonical_schema() schema = json.loads(self.schema_path.read_text(encoding="utf-8")) schema["$schema"] = "https://json-schema.org/draft/2019-09/schema" self.schema_path.write_text(json.dumps(schema), encoding="utf-8") drifted_schema = self.run_validator(["FR-46"], install_schema=False) self.assertNotEqual(drifted_schema.returncode, 0) self.assertIn("error[INVALID_SCHEMA_CONTRACT]", drifted_schema.stderr) def test_rejects_duplicate_nonstandard_recursive_and_large_integer_json(self) -> None: malformed_documents = { "duplicate member": ( '{"schema_version":1,"schema_version":1,' '"product":"crank-community","flows":[]}', "INVALID_JSON", ), "non-standard number": ( '{"schema_version":1,"product":NaN,"flows":[]}', "INVALID_JSON", ), "recursive": ("[" * 100_000 + "0" + "]" * 100_000, "INVALID_JSON"), "large integer": ("9" * 5_000, "INVALID_JSON"), } for label, (content, expected_code) in malformed_documents.items(): with self.subTest(label=label): self.inventory_path.write_text(content, encoding="utf-8") result = self.run_validator(["FR-46"]) self.assertNotEqual(result.returncode, 0) self.assertIn(f"error[{expected_code}]", result.stderr) self.assertNotIn("Traceback", result.stderr) self.assertLessEqual(len(result.stderr.encode("utf-8")), 64 * 1024) def test_schema_parity_rejects_required_type_ref_and_pattern_drift(self) -> None: self.write_inventory([self.valid_flow("schema-parity", "api", "FR-46")]) mutations = { "root required missing": lambda schema: schema.pop("required"), "root contract inversion": lambda schema: schema.update({"not": {}}), "malformed evidence required": lambda schema: schema["$defs"]["evidence"].update( {"required": [{}]} ), "flow ref drift": lambda schema: schema["properties"]["flows"]["items"].update( {"$ref": "#/$defs/evidence"} ), "owner type drift": lambda schema: schema["$defs"]["flow"]["properties"][ "owner" ].update({"type": "integer"}), "owner const drift": lambda schema: schema["$defs"]["flow"]["properties"][ "owner" ].update({"const": "impossible"}), "nested applicator drift": lambda schema: schema["$defs"]["evidencePaths"][ "items" ].update({"anyOf": [{"type": "integer"}]}), "evidence pattern drift": lambda schema: schema["$defs"]["evidencePaths"][ "items" ].update({"pattern": ".+"}), } for label, mutate in mutations.items(): with self.subTest(label=label): self.install_canonical_schema() schema = json.loads(self.schema_path.read_text(encoding="utf-8")) mutate(schema) self.schema_path.write_text(json.dumps(schema), encoding="utf-8") result = self.run_validator(["FR-46"], install_schema=False) self.assertNotEqual(result.returncode, 0) self.assertIn("error[INVALID_SCHEMA_CONTRACT]", result.stderr) self.assertNotIn("Traceback", result.stderr) self.install_canonical_schema() schema = json.loads(self.schema_path.read_text(encoding="utf-8")) schema["$defs"]["flow"]["properties"]["status"]["enum"].reverse() self.schema_path.write_text(json.dumps(schema), encoding="utf-8") reordered_enum = self.run_validator(["FR-46"], install_schema=False) self.assertEqual(reordered_enum.returncode, 0, reordered_enum.stderr) def test_rejects_boolean_schema_version_and_whitespace_required_text(self) -> None: base = self.valid_flow("strict-scalars", "api", "FR-46") payload_cases: list[tuple[str, dict[str, object], str]] = [] boolean_version = { "schema_version": True, "product": "crank-community", "flows": [copy.deepcopy(base)], } payload_cases.append(("boolean inventory version", boolean_version, "MISSING_REQUIRED_FIELD")) whitespace_owner = copy.deepcopy(base) whitespace_owner["owner"] = " \t\n " payload_cases.append( ( "whitespace owner", {"schema_version": 1, "product": "crank-community", "flows": [whitespace_owner]}, "MISSING_OWNER", ) ) whitespace_outcome = copy.deepcopy(base) whitespace_outcome["user_outcome"] = " \t\n " payload_cases.append( ( "whitespace outcome", {"schema_version": 1, "product": "crank-community", "flows": [whitespace_outcome]}, "LIMIT_EXCEEDED", ) ) for label, payload, expected_code in payload_cases: with self.subTest(label=label): self.inventory_path.write_text(json.dumps(payload), encoding="utf-8") result = self.run_validator(["FR-46"]) self.assertNotEqual(result.returncode, 0) self.assertIn(f"error[{expected_code}]", result.stderr) self.write_inventory([base]) self.install_canonical_schema() schema = json.loads(self.schema_path.read_text(encoding="utf-8")) schema["properties"]["schema_version"]["const"] = True self.schema_path.write_text(json.dumps(schema), encoding="utf-8") schema_result = self.run_validator(["FR-46"], install_schema=False) self.assertNotEqual(schema_result.returncode, 0) self.assertIn("error[INVALID_SCHEMA_CONTRACT]", schema_result.stderr) def test_diagnostic_accumulation_is_capped_counted_and_redacted(self) -> None: secret_canary = "SECRET_CANARY_unknown-field" flow = self.valid_flow("bounded-diagnostics", "api", "FR-46") for index in range(2_000): flow[f"{secret_canary}-{index:04d}"] = True self.write_inventory([flow]) result = self.run_validator(["FR-46"]) self.assertNotEqual(result.returncode, 0) self.assertIn("error[TRUNCATED] /: diagnostics omitted=1000", result.stderr) self.assertEqual(result.stderr.count("error[UNKNOWN_FIELD]"), 1_000) self.assertNotIn(secret_canary, result.stderr) self.assertLessEqual(len(result.stderr.encode("utf-8")), 64 * 1024) def test_rejects_oversized_document_and_field_limits(self) -> None: self.inventory_path.write_bytes(b" " * (4_194_304 + 1)) oversized = self.run_validator(["FR-46"]) self.assertNotEqual(oversized.returncode, 0) self.assertIn("error[INPUT_TOO_LARGE]", oversized.stderr) flow = self.valid_flow("bounded-flow", "api", "FR-46") flow["user_outcome"] = "x" * 4_097 self.write_inventory([flow]) overlong = self.run_validator(["FR-46"]) self.assertNotEqual(overlong.returncode, 0) self.assertIn("error[LIMIT_EXCEEDED]", overlong.stderr) def test_named_string_limits_cover_below_at_and_above_boundaries(self) -> None: string_cases = [ ("id", 128, lambda length: "a" * length), ("owner", 256, lambda length: "o" * length), ("user_outcome", 4_096, lambda length: "u" * length), ("notes", 4_096, lambda length: "n" * length), ("requirements", 128, lambda length: "FR-" + "1" * (length - 3)), ] for field, limit, make_value in string_cases: for delta in (-1, 0, 1): with self.subTest(field=field, length=limit + delta): safe_field = field.replace("_", "-") flow = self.valid_flow(f"bounded-{safe_field}-{delta + 1}", "api", "FR-46") value = make_value(limit + delta) if field == "requirements": flow[field] = [value] required_frs = [value] else: flow[field] = value required_frs = ["FR-46"] self.write_inventory([flow]) result = self.run_validator(required_frs) if delta <= 0: self.assertEqual(result.returncode, 0, result.stderr) else: self.assertNotEqual(result.returncode, 0) self.assertIn("error[LIMIT_EXCEEDED]", result.stderr) def test_collection_limits_cover_below_at_and_above_boundaries(self) -> None: module = load_validator_module() automated = self.write_evidence("evidence/shared/automated.txt") manual = self.write_evidence("evidence/shared/manual.md") def direct_flow(index: int) -> dict[str, object]: return { "id": f"flow-{index:05d}", "type": "api", "requirements": ["FR-46"], "user_outcome": "A bounded flow succeeds.", "owner": "community-release", "status": "implemented", "capabilities": ["tools"], "evidence": {"automated": [automated], "manual": [manual]}, } all_flows = [direct_flow(index) for index in range(10_001)] for count in (9_999, 10_000, 10_001): with self.subTest(collection="flows", count=count): inventory = { "schema_version": 1, "product": "crank-community", "flows": all_flows[:count], } diagnostics, _ = module.validate_inventory(self.root, inventory, ["FR-46"]) has_limit_error = any( item.code == "LIMIT_EXCEEDED" and item.pointer == "/flows" for item in diagnostics.ordered() ) self.assertEqual(has_limit_error, count > 10_000) for count in (63, 64, 65): with self.subTest(collection="requirements", count=count): flow = self.valid_flow(f"requirements-{count}", "api", "FR-1") flow["requirements"] = [f"FR-{index}" for index in range(1, count + 1)] self.write_inventory([flow]) result = self.run_validator(["FR-1"]) self.assertEqual(result.returncode == 0, count <= 64, result.stderr) evidence_paths = [ self.write_evidence(f"evidence/bounds/path-{index:02d}.txt") for index in range(65) ] for count in (63, 64, 65): with self.subTest(collection="evidence", count=count): flow = self.valid_flow(f"evidence-{count}", "api", "FR-46") flow["evidence"] = { "automated": evidence_paths[:count], "manual": [manual], } self.write_inventory([flow]) result = self.run_validator(["FR-46"]) self.assertEqual(result.returncode == 0, count <= 64, result.stderr) for count in (15, 16, 17): with self.subTest(collection="capability-bound", count=count): diagnostics = module.validate_string_list( [f"capability-{index}" for index in range(count)], "/capabilities", 16, 128, ) has_limit_error = any(item.code == "LIMIT_EXCEEDED" for item in diagnostics) self.assertEqual(has_limit_error, count > 16) def test_evidence_path_limit_covers_below_at_and_above_boundary(self) -> None: def path_of_length(length: int) -> str: segment_count = (length + 199) // 201 character_count = length - (segment_count - 1) lengths: list[int] = [] remaining = character_count for index in range(segment_count): slots_left = segment_count - index - 1 current = min(200, remaining - slots_left) lengths.append(current) remaining -= current path = "/".join("p" * size for size in lengths) self.assertEqual(len(path), length) return path manual = self.write_evidence("evidence/manual/path-limit.md") for length in (1_023, 1_024, 1_025): with self.subTest(length=length): evidence_path = path_of_length(length) if length <= 1_024: self.write_evidence(evidence_path) flow = self.valid_flow(f"path-limit-{length}", "api", "FR-46") flow["evidence"] = { "automated": [evidence_path], "manual": [manual], } self.write_inventory([flow]) result = self.run_validator(["FR-46"]) self.assertEqual(result.returncode == 0, length <= 1_024, result.stderr) def test_semantic_validator_enforces_schema_shape_uniqueness_and_fr_format(self) -> None: cases: list[tuple[str, object, str]] = [] unknown_top = { "schema_version": 1, "product": "crank-community", "flows": [self.valid_flow("unknown-top", "api", "FR-46")], "unexpected": True, } cases.append(("unknown top-level field", unknown_top, "UNKNOWN_FIELD")) unknown_flow = self.valid_flow("unknown-flow", "api", "FR-46") unknown_flow["unexpected"] = True cases.append( ( "unknown flow field", {"schema_version": 1, "product": "crank-community", "flows": [unknown_flow]}, "UNKNOWN_FIELD", ) ) duplicate_requirement = self.valid_flow("duplicate-fr", "api", "FR-46") duplicate_requirement["requirements"] = ["FR-46", "FR-46"] cases.append( ( "duplicate requirement", { "schema_version": 1, "product": "crank-community", "flows": [duplicate_requirement], }, "DUPLICATE_LIST_ITEM", ) ) malformed_requirement = self.valid_flow("malformed-fr", "api", "FR-46") malformed_requirement["requirements"] = ["FR-0"] cases.append( ( "malformed requirement", { "schema_version": 1, "product": "crank-community", "flows": [malformed_requirement], }, "INVALID_FORMAT", ) ) unknown_evidence = self.valid_flow("unknown-evidence", "api", "FR-46") evidence = unknown_evidence["evidence"] self.assertIsInstance(evidence, dict) evidence["unexpected"] = [] cases.append( ( "unknown evidence field", { "schema_version": 1, "product": "crank-community", "flows": [unknown_evidence], }, "UNKNOWN_FIELD", ) ) for label, payload, expected_code in cases: with self.subTest(label=label): self.inventory_path.write_text( json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) result = self.run_validator(["FR-46"]) self.assertNotEqual(result.returncode, 0) self.assertIn(f"error[{expected_code}]", result.stderr) def test_accepts_planned_modern_mcp_and_load_run_capabilities(self) -> None: capabilities = ["resources", "prompts", "tasks", "load_runs"] flows = [ self.valid_flow( f"planned-{capability.replace('_', '-')}", "mcp" if capability != "load_runs" else "ui", f"FR-{index}", "planned", [capability], ) for index, capability in enumerate(capabilities, start=47) ] self.write_inventory(flows) result = self.run_validator([f"FR-{index}" for index in range(47, 51)]) self.assertEqual(result.returncode, 0, result.stderr) for capability in capabilities: self.assertNotIn(f"error[FORBIDDEN_COMMUNITY_CAPABILITY]: {capability}", result.stderr) def test_rejects_forbidden_community_capabilities_despite_allow_text(self) -> None: forbidden = [ "multi_workspace", # community-scope: allow=multi-workspace "enterprise_rbac", # community-scope: allow=enterprise "sso", # community-scope: allow=sso "non_rest_upstream", # community-scope: allow=non-rest-upstream "arbitrary_distributed_load_targets", # community-scope: allow=distributed-load-targets ] for index, capability in enumerate(forbidden, start=1): with self.subTest(capability=capability): flow = self.valid_flow( f"forbidden-{index}", "api", "FR-46", "planned", [capability], ) flow["notes"] = f"community-scope: allow={capability}" self.write_inventory([flow]) result = self.run_validator(["FR-46"]) self.assertNotEqual(result.returncode, 0) self.assertIn("error[FORBIDDEN_COMMUNITY_CAPABILITY]", result.stderr) self.assertIn(capability, result.stderr) def test_scope_scanner_receives_inventory_files_explicitly_and_cannot_be_bypassed(self) -> None: clean_inventory = self.valid_flow( "planned-resources", "mcp", "FR-47", "planned", ["resources"], ) self.write_inventory([clean_inventory]) explicit_files = [ str(self.inventory_path.relative_to(self.root)), "docs/new-untracked-scope-note.md", ] note = self.root / explicit_files[1] note.write_text("Community Resources remain planned.\n", encoding="utf-8") clean = subprocess.run( [ sys.executable, str(COMMUNITY_CHECKER), "--root", str(self.root), "--files", *explicit_files, ], check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) self.assertEqual(clean.returncode, 0, clean.stderr) self.assertIn("2 files scanned", clean.stdout) note.write_text( "community-scope: allow=enterprise\n" "enterprise_rbac remains outside Community scope.\n", # community-scope: allow=enterprise encoding="utf-8", ) rejected = subprocess.run( [ sys.executable, str(COMMUNITY_CHECKER), "--root", str(self.root), "--files", *explicit_files, ], check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) self.assertNotEqual(rejected.returncode, 0) self.assertIn("docs/new-untracked-scope-note.md:2", rejected.stderr) self.assertIn("forbidden marker", rejected.stderr) self.assertIn("enterprise", rejected.stderr) # community-scope: allow=enterprise def test_canonical_inventory_seed_does_not_promote_planned_work(self) -> None: for path in [ CANONICAL_INVENTORY, CANONICAL_SCHEMA, ]: self.assertTrue(path.is_file(), f"Missing canonical artifact: {path}") inventory = json.loads(CANONICAL_INVENTORY.read_text(encoding="utf-8")) schema = json.loads(CANONICAL_SCHEMA.read_text(encoding="utf-8")) flow_schema = schema["$defs"]["flow"] evidence_paths = schema["$defs"]["evidencePaths"] self.assertEqual( schema["$schema"], "https://json-schema.org/draft/2020-12/schema", ) self.assertEqual(schema["properties"]["flows"]["maxItems"], 10_000) self.assertEqual( set(flow_schema["properties"]["type"]["enum"]), {"ui", "api", "mcp"}, ) self.assertEqual( set(flow_schema["properties"]["status"]["enum"]), {"implemented", "planned", "gap", "blocked"}, ) self.assertEqual(evidence_paths["minItems"], 1) self.assertEqual(evidence_paths["maxItems"], 64) target_capabilities = {"resources", "prompts", "tasks", "load_runs"} matching_flows = [ flow for flow in inventory["flows"] if target_capabilities.intersection(flow.get("capabilities", [])) ] observed_capabilities = { capability for flow in matching_flows for capability in flow.get("capabilities", []) if capability in target_capabilities } self.assertEqual(observed_capabilities, target_capabilities) self.assertTrue(matching_flows) self.assertTrue( all(flow["status"] != "implemented" for flow in matching_flows), "Story 1.1 must not claim target capabilities are implemented.", ) def test_canonical_inventory_covers_all_requirements_and_required_surfaces(self) -> None: inventory = json.loads(CANONICAL_INVENTORY.read_text(encoding="utf-8")) required_surfaces = json.loads(REQUIRED_SURFACES.read_text(encoding="utf-8")) flows = {flow["id"]: flow for flow in inventory["flows"]} observed_requirements = { requirement for flow in flows.values() for requirement in flow["requirements"] } self.assertEqual( {f"FR-{number}" for number in range(1, 55)} - observed_requirements, set(), ) self.assertEqual( set(required_surfaces["required_flow_ids"]) - set(flows), set(), ) for flow_id in required_surfaces["required_flow_ids"]: flow = flows[flow_id] self.assertIn(flow["status"], {"implemented", "blocked"}) for evidence_kind in ("automated", "manual"): for evidence_path in flow["evidence"][evidence_kind]: resolved = ROOT / evidence_path self.assertTrue(resolved.is_file(), evidence_path) self.assertFalse(resolved.is_symlink(), evidence_path) def test_tracked_documentation_describes_planned_scope(self) -> None: for documentation in [ COMMUNITY_README, DOCS_INDEX, COMMUNITY_INTRO, COMMUNITY_README_EN, ]: text = documentation.read_text(encoding="utf-8") self.assertIn("docs/capability-inventory.json", text) self.assertRegex(text.lower(), r"\b(planned|gap|blocked)\b") def test_no_story_acceptance_scenario_remains_skipped(self) -> None: for name, method in vars(type(self)).items(): if not name.startswith("test_") or name == self._testMethodName: continue self.assertFalse( getattr(method, "__unittest_skip__", False), f"Story acceptance scenario remains skipped: {name}", ) def test_canonical_validator_is_a_ci_and_release_prerequisite(self) -> None: justfile = JUSTFILE.read_text(encoding="utf-8") ci = CI_WORKFLOW.read_text(encoding="utf-8") release = RELEASE_WORKFLOW.read_text(encoding="utf-8") command = "scripts/validate-capability-inventory.py" self.assertIn("capability-inventory-check:", justfile) self.assertIn("just capability-inventory-check", justfile) self.assertLess(ci.index("Run tooling unit tests"), ci.index(command)) self.assertLess(ci.index(command), ci.index("Check Community scope")) self.assertIn("python3 --version", release) self.assertLess(release.index("Run tooling unit tests"), release.index(command)) self.assertLess(release.index(command), release.index("Check Community scope")) self.assertLess(release.index("Check Community scope"), release.index("Build release binaries")) # --- Story 3.1 ATDD (red): the approved two-revision MCP window ------------------------------- # Oracle: tests/fixtures/mcp-revisions/* (owner-approved 2026-10-11 contract, source-backed, not a # snapshot of current output). Behaviour is observed on the real validator # (scripts/validate-capability-inventory.py) and the real schema copy. # The protocol vocabulary here (supported/not_applicable/experimental/deferred) is deliberately # separate from the unchanged flow vocabulary (implemented/planned/gap/blocked). PROTOCOL_STATUSES = {"supported", "not_applicable", "experimental", "deferred"} def load_revision_fixture(self, name: str) -> dict[str, object]: return json.loads((MCP_REVISION_FIXTURES / name).read_text(encoding="utf-8")) def write_revision_inventory(self, name: str) -> tuple[dict[str, object], dict[str, object]]: payload = self.load_revision_fixture(name) expected = payload.pop("expected", {}) for flow in payload.get("flows", []): # type: ignore[union-attr] for kind in ("automated", "manual"): for relative in flow.get("evidence", {}).get(kind, []): self.write_evidence(relative) self.inventory_path.write_text( json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) return payload, expected # type: ignore[return-value] def required_surfaces(self) -> set[str]: oracle = self.load_revision_fixture("source-backed-capabilities.json") return { item["surface"] for item in oracle["items"] # type: ignore[index] if item["role"] == "server" } def test_at_3_1_01_full_reviewed_candidate_passes_the_validity_gate(self) -> None: """Target oracle: the FULL reviewed candidate is an accepted v2 inventory (Gate A, exit 0). This is a genuine behavioural RED on the current revision: the validator has no v2 `protocol` block, so it answers `schema_version 1 is required` / `UNKNOWN_FIELD`. The assertions below describe the agreed target behaviour and are deliberately NOT relaxed to the current one. """ self.write_candidate_inventory() result = self.run_validator(["FR-13"]) combined = result.stdout + result.stderr self.assertEqual( result.returncode, 0, "the complete reviewed v2 candidate must pass the validity gate, observed: " + combined, ) # The agreed validity output is the existing one-line summary plus the protocol counts. self.assertIn("Capability inventory validation passed", combined) self.assertRegex(combined, r"\bprotocol_items=50\b") self.assertRegex(combined, r"\brevisions=2\b") def test_at_3_1_01_scoped_pre_atdd_candidate_stays_a_scoped_input(self) -> None: """Preserved pre-ATDD input: the scoped window candidate is NOT promoted to full validity. Pure oracle-input provenance - no validator run, so this is not labelled validator GREEN. The file keeps its own recorded verdict (`counts_as_exhaustive: false`, completeness BLOCKED) and stays byte-identical (checked by the ATDD snapshot verifier); the reviewed candidate above is the completeness basis. """ payload, expected = self.write_revision_inventory("inventory-two-revision-window.json") protocol: Any = payload["protocol"] cells = protocol["capabilities"] self.assertEqual(payload["schema_version"], 2) self.assertEqual( [entry["revision"] for entry in protocol["revisions"]], ["2026-07-28", "2025-11-25"], ) self.assertEqual( set(entry["revision"] for entry in protocol["revisions"]) & set(protocol["withdrawn"]), set(), ) self.assertIn("scope_note", payload) self.assertEqual(expected["gate"], "validity-shape") self.assertEqual(expected["verdict"], "scoped-candidate-not-full-validity") self.assertEqual(expected["completeness"], "blocked-scoped-subset") self.assertEqual({cell["surface"] for cell in cells}, self.required_surfaces()) def test_at_3_1_01_third_revision_outside_the_approved_window_is_rejected_by_name(self) -> None: """Bounded one-factor mutation OF THE REVIEWED CANDIDATE: one extra ledger revision entry.""" payload = self.write_candidate_inventory("inventory-mutation-third-revision.json") revisions = [entry["revision"] for entry in payload["protocol"]["revisions"]] # type: ignore[index] self.assertEqual(revisions[:2], ["2026-07-28", "2025-11-25"]) self.assertEqual(revisions[2:], ["2025-03-26"], "only one revision entry may be added") result = self.run_validator(["FR-13"]) combined = result.stdout + result.stderr self.assertNotEqual(result.returncode, 0, result.stdout) self.assertIn("2025-03-26", combined) def test_at_3_1_03_reviewed_candidate_omission_is_rejected_by_the_triple(self) -> None: """Bounded one-factor mutation OF THE REVIEWED CANDIDATE: exactly one reviewed row dropped.""" payload = self.write_candidate_inventory("inventory-mutation-omission.json") observed = { (cell["role"], cell["kind"], cell["surface"]) for cell in payload["protocol"]["capabilities"] # type: ignore[index] } self.assertEqual(len(observed), 49, "only one reviewed identity may be dropped") self.assertNotIn(("client", "method", "prompts/get"), observed) result = self.run_validator(["FR-13"]) combined = result.stdout + result.stderr self.assertNotEqual(result.returncode, 0, result.stdout) lines = [line for line in combined.splitlines() if "OMISSION" in line] self.assertTrue( any("prompts/get" in line for line in lines), f"the omitted triple must be printed by name: {combined}", ) self.assertTrue( any("client" in line and "method" in line for line in lines), f"the full (role, kind, surface) triple must be printed: {combined}", ) def test_at_3_1_03_undocumented_surface_is_rejected_by_the_triple(self) -> None: """Bounded one-factor mutation OF THE REVIEWED CANDIDATE: one extra undocumented triple.""" payload = self.write_candidate_inventory("inventory-mutation-undocumented-surface.json") observed = { (cell["role"], cell["kind"], cell["surface"]) for cell in payload["protocol"]["capabilities"] # type: ignore[index] } self.assertEqual(len(observed), 51, "only one extra identity may be added") self.assertIn(("server", "method", "resources/read"), observed) result = self.run_validator(["FR-13"]) combined = result.stdout + result.stderr self.assertNotEqual(result.returncode, 0, result.stdout) lines = [line for line in combined.splitlines() if "UNDOCUMENTED_SURFACE" in line] self.assertTrue( any("resources/read" in line for line in lines), f"the undocumented triple must be printed by name: {combined}", ) self.assertTrue( any("server" in line and "method" in line for line in lines), f"the full (role, kind, surface) triple must be printed: {combined}", ) def test_at_3_1_01_revision_missing_only_the_digest_is_rejected_by_that_field(self) -> None: """One-factor provenance negative: only `provenance.sha256` of the modern revision is gone.""" payload = self.write_candidate_inventory("inventory-mutation-missing-digest.json") modern = next( entry for entry in payload["protocol"]["revisions"] # type: ignore[index] if entry["revision"] == "2026-07-28" ) self.assertEqual(sorted(modern["provenance"]), ["captured_on", "locator"]) result = self.run_validator(["FR-13"]) combined = result.stdout + result.stderr self.assertNotEqual(result.returncode, 0, result.stdout) diagnostics = [line for line in combined.splitlines() if line.startswith("error[")] digest_lines = [line for line in diagnostics if "sha256" in line] self.assertTrue( any("2026-07-28" in line for line in digest_lines), f"the missing digest must be reported for the revision that lost it: {combined}", ) self.assertFalse( any("captured_on" in line for line in diagnostics), f"the date is still present and must not be reported missing: {combined}", ) def test_at_3_1_01_revision_missing_only_the_date_is_rejected_by_that_field(self) -> None: """One-factor provenance negative: only `provenance.captured_on` of the modern revision is gone.""" payload = self.write_candidate_inventory("inventory-mutation-missing-date.json") modern = next( entry for entry in payload["protocol"]["revisions"] # type: ignore[index] if entry["revision"] == "2026-07-28" ) self.assertEqual(sorted(modern["provenance"]), ["locator", "sha256"]) result = self.run_validator(["FR-13"]) combined = result.stdout + result.stderr self.assertNotEqual(result.returncode, 0, result.stdout) diagnostics = [line for line in combined.splitlines() if line.startswith("error[")] date_lines = [line for line in diagnostics if "captured_on" in line] self.assertTrue( any("2026-07-28" in line for line in date_lines), f"the missing capture date must be reported for the revision that lost it: {combined}", ) self.assertFalse( any("sha256" in line for line in diagnostics), f"the digest is still present and must not be reported missing: {combined}", ) def test_at_3_1_03_ordered_validity_first_gate_table_is_exercised_when_mode_exists(self) -> None: """The agreed validity-first gate table (A0 / B3 / B0 / invalid1) on the real CLI. While the CLI exposes no `--mode`, this stays an honest interface gap (skip, never a synthetic red). As soon as `--mode` exists the same test body exercises every agreed row, so a wrong Gate A/B implementation fails here instead of being skipped. """ probe = subprocess.run( [sys.executable, str(VALIDATOR), "--help"], check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) help_text = probe.stdout + probe.stderr self.assertEqual(probe.returncode, 0, help_text) if "--mode" not in help_text: self.skipTest( "interface gap, not a behavioural red: the validator CLI exposes no --mode yet, so " "the ordered validity/feature-complete gate table cannot be exercised on this " "revision" ) self.install_canonical_schema() # invalid1 - structurally invalid input: `invalid` in BOTH modes, step-0 diagnostics, and no # feature-complete evaluation at all (accepted spec `remaining-ac-spec.md:392`). self.inventory_path.write_text("{not-json", encoding="utf-8") for mode in ("validity", "feature-complete"): with self.subTest(row="structural-invalid", mode=mode): result = self.run_validator(["FR-13"], install_schema=False, mode=mode) combined = result.stdout + result.stderr self.assertEqual(result.returncode, 1, combined) self.assertIn("status=invalid", combined) self.assertIn("error[", combined) self.assertNotIn("status=blocked", combined) self.assertNotIn("status=complete", combined) self.assertNotRegex(combined, r"blockers=[1-9]") # invalid1 - semantic validity diagnostics (only the modern `provenance.sha256` is gone): # `invalid` in BOTH modes, the targeted missing-digest diagnostic, no feature-complete output # (accepted spec `remaining-ac-spec.md:393`). self.write_candidate_inventory("inventory-mutation-missing-digest.json") for mode in ("validity", "feature-complete"): with self.subTest(row="semantic-invalid", mode=mode): result = self.run_validator(["FR-13"], mode=mode) combined = result.stdout + result.stderr self.assertEqual(result.returncode, 1, combined) self.assertIn("status=invalid", combined) diagnostics = [line for line in combined.splitlines() if line.startswith("error[")] self.assertTrue( any("2026-07-28" in line and "sha256" in line for line in diagnostics), f"the missing digest must be named for the revision that lost it: {combined}", ) self.assertNotIn("status=blocked", combined) self.assertNotIn("status=complete", combined) self.assertNotRegex(combined, r"blockers=[1-9]") # invalid1 - the rejected false-not-applicable variant: applicable scheduled work renamed to # `not_applicable` must stay `invalid` in BOTH modes, never `complete` with 0 blockers. self.write_candidate_inventory("inventory-mutation-false-not-applicable.json") for mode in ("validity", "feature-complete"): with self.subTest(row="false-not-applicable", mode=mode): result = self.run_validator(["FR-13"], mode=mode) combined = result.stdout + result.stderr self.assertEqual(result.returncode, 1, combined) self.assertIn("status=invalid", combined) self.assertNotIn("status=blocked", combined) self.assertNotIn("status=complete", combined) self.assertNotRegex(combined, r"blockers=[1-9]") # A0 / B3 - a truthful candidate passes validity and is blocked by the feature gate. self.write_candidate_inventory() validity = self.run_validator(["FR-13"], mode="validity") self.assertEqual(validity.returncode, 0, validity.stdout + validity.stderr) self.assertIn("Capability inventory validation passed", validity.stdout + validity.stderr) feature_complete = self.run_validator(["FR-13"], mode="feature-complete") self.assertEqual( feature_complete.returncode, 3, feature_complete.stdout + feature_complete.stderr, ) self.assertIn("status=blocked", feature_complete.stdout + feature_complete.stderr) self.assertRegex( feature_complete.stdout + feature_complete.stderr, r"blockers=[1-9][0-9]*", ) # B0 - the explicitly SYNTHETIC no-blocker input (hypothetical completed state, marked with # its own owner) is complete in the feature gate. self.write_candidate_inventory("inventory-synthetic-no-blocker.json") validity = self.run_validator(["FR-13"], mode="validity") self.assertEqual(validity.returncode, 0, validity.stdout + validity.stderr) self.assertIn("Capability inventory validation passed", validity.stdout + validity.stderr) feature_complete = self.run_validator(["FR-13"], mode="feature-complete") self.assertEqual( feature_complete.returncode, 0, feature_complete.stdout + feature_complete.stderr, ) self.assertIn("status=complete", feature_complete.stdout + feature_complete.stderr) self.assertRegex( feature_complete.stdout + feature_complete.stderr, r"blockers=0\b", ) # --- Story 3.1 remaining-foundation ATDD additions ------------------------------------------ # Oracle inputs: the reviewed v2 candidate (`inventory-reviewed-v2-candidate.json`), its bounded # one-factor mutations, and the independent per-era frozen projections. The reviewed identity is # the triple `(role, kind, surface)`; `role` is one enum {server, client, both} with a # kind-specific meaning (declarer for capability/extension, sender for method/notification). # The target validator behaviour (v2 protocol block, Gate A/B split, named diagnostics) does not # exist on this revision, so the assertions below are the contract the implementation stage must # satisfy. Anything that can only fail because a CLI flag or a module is absent is recorded as an # honest API gap, never as a behavioural red and never as a pass. REVIEWED_IDENTITY_TRIPLES = { # server capabilities (surface-inventory.md section 3) ("server", "capability", "experimental"), ("server", "capability", "logging"), ("server", "capability", "completions"), ("server", "capability", "prompts"), ("server", "capability", "resources"), ("server", "capability", "tools"), ("server", "capability", "extensions"), ("server", "capability", "tasks"), # client capabilities (section 4) ("client", "capability", "experimental"), ("client", "capability", "roots"), ("client", "capability", "sampling"), ("client", "capability", "elicitation"), ("client", "capability", "extensions"), ("client", "capability", "tasks"), # methods (section 5); role = the sending party ("client", "method", "tools/list"), ("client", "method", "tools/call"), ("client", "method", "resources/list"), ("client", "method", "resources/read"), ("client", "method", "resources/templates/list"), ("client", "method", "resources/subscribe"), ("client", "method", "resources/unsubscribe"), ("client", "method", "prompts/list"), ("client", "method", "prompts/get"), ("client", "method", "completion/complete"), ("client", "method", "logging/setLevel"), ("client", "method", "initialize"), ("both", "method", "ping"), ("client", "method", "server/discover"), ("client", "method", "subscriptions/listen"), ("server", "method", "roots/list"), ("server", "method", "sampling/createMessage"), ("server", "method", "elicitation/create"), ("both", "method", "tasks/get"), ("both", "method", "tasks/list"), ("both", "method", "tasks/result"), ("both", "method", "tasks/cancel"), # notifications (section 6) ("server", "notification", "notifications/tools/list_changed"), ("server", "notification", "notifications/resources/list_changed"), ("server", "notification", "notifications/prompts/list_changed"), ("server", "notification", "notifications/resources/updated"), ("server", "notification", "notifications/progress"), ("server", "notification", "notifications/message"), ("server", "notification", "notifications/subscriptions/acknowledged"), ("both", "notification", "notifications/cancelled"), ("client", "notification", "notifications/initialized"), ("client", "notification", "notifications/roots/list_changed"), ("both", "notification", "notifications/tasks/status"), ("server", "notification", "notifications/elicitation/complete"), # extensions (section 7) ("server", "extension", "io.modelcontextprotocol/tasks"), ("client", "extension", "io.modelcontextprotocol/oauth-client-credentials"), } REVIEWED_ROLES = {"server", "client", "both"} REVIEWED_KINDS = {"capability", "method", "notification", "extension"} REVIEWED_STATUSES = {"supported", "not_applicable", "experimental", "deferred"} # Independent per-era ABSENCE oracle, read off the "absent" cells of the reviewed tables in # `build/remaining-ac-spec/surface-inventory.md` sections 3-7 (deliberately NOT taken from the # fixture). Rule O2 resolves an era in two steps (`remaining-ac-spec.md:296-300`): a surface whose # `revisions[]` does not contain the era is ABSENT for that era - never `not_applicable` - and only # then is the shallow `per_revision[era]` merge applied. Applicable per-era completeness is # therefore `REVIEWED_IDENTITY_TRIPLES - REVIEWED_ABSENT_`, not the 50-identity default union. REVIEWED_ABSENT_MODERN = { # sections 3/4: modern expresses Tasks as an extension, so the server `tasks` capability is # absent, and the client `tasks` declaration does not exist in modern either ("server", "capability", "tasks"), ("client", "capability", "tasks"), # section 5: method literals absent from the pinned modern schema ("client", "method", "resources/subscribe"), ("client", "method", "resources/unsubscribe"), ("client", "method", "logging/setLevel"), ("client", "method", "initialize"), ("both", "method", "ping"), ("both", "method", "tasks/get"), ("both", "method", "tasks/list"), ("both", "method", "tasks/result"), ("both", "method", "tasks/cancel"), # section 6: notifications absent from the pinned modern schema ("client", "notification", "notifications/initialized"), ("client", "notification", "notifications/roots/list_changed"), ("both", "notification", "notifications/tasks/status"), ("server", "notification", "notifications/elicitation/complete"), } REVIEWED_ABSENT_LEGACY = { # sections 3/4: neither legacy `ServerCapabilities` nor legacy `ClientCapabilities` has an # `extensions` key ("server", "capability", "extensions"), ("client", "capability", "extensions"), # sections 5/6: modern-only method literals and the subscription acknowledgement ("client", "method", "server/discover"), ("client", "method", "subscriptions/listen"), ("server", "notification", "notifications/subscriptions/acknowledged"), # section 7: both extension identifiers are declared in the modern schema only ("server", "extension", "io.modelcontextprotocol/tasks"), ("client", "extension", "io.modelcontextprotocol/oauth-client-credentials"), } def load_candidate_fixture(self, name: str = "inventory-reviewed-v2-candidate.json") -> dict[str, object]: return self.load_revision_fixture(name) def write_candidate_inventory(self, name: str = "inventory-reviewed-v2-candidate.json") -> dict[str, object]: """Install a reviewed candidate/mutation fixture as the validator input.""" payload = self.load_candidate_fixture(name) for flow in payload.get("flows", []): # type: ignore[union-attr] for kind in ("automated", "manual"): for relative in flow.get("evidence", {}).get(kind, []): self.write_evidence(relative) self.inventory_path.write_text( json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) return payload def effective_entry(self, entry: dict[str, object], revision: str) -> dict[str, object]: merged = {key: value for key, value in entry.items() if key != "per_revision"} overrides = entry.get("per_revision", {}).get(revision, {}) # type: ignore[union-attr] merged.update(overrides) return merged def test_at_3_1_03_reviewed_candidate_is_identity_complete_and_metadata_stays_separate(self) -> None: """The reviewed completeness basis itself: one judged entry per reviewed row, no more, no less.""" payload = self.load_candidate_fixture() protocol: Any = payload["protocol"] self.assertEqual(payload["schema_version"], 2) self.assertEqual( [entry["revision"] for entry in protocol["revisions"]], ["2026-07-28", "2025-11-25"], ) for entry in protocol["revisions"]: provenance = entry["provenance"] self.assertTrue(provenance["locator"] and provenance["sha256"] and provenance["captured_on"]) self.assertEqual(protocol["withdrawn"], ["2025-06-18", "2025-03-26"]) capabilities = protocol["capabilities"] triples = [(cell["role"], cell["kind"], cell["surface"]) for cell in capabilities] self.assertEqual(len(triples), len(set(triples)), "identity is the (role, kind, surface) triple") self.assertEqual(set(triples), self.REVIEWED_IDENTITY_TRIPLES) for cell in capabilities: self.assertIn(cell["role"], self.REVIEWED_ROLES) self.assertIn(cell["kind"], self.REVIEWED_KINDS) self.assertIn(cell["status"], self.REVIEWED_STATUSES) self.assertTrue(cell["owner"] and cell["fallback"] and cell["evidence_target"]) if cell["status"] == "supported": self.assertNotIn("reason", cell) else: self.assertTrue(cell["reason"], cell) # `io.modelcontextprotocol/related-task` is a non-advertisable `_meta` metadata note, not a # capability extension: it must stay out of the four-kind ledger and out of the advertisement. self.assertFalse( any( cell["surface"] == "io.modelcontextprotocol/related-task" for cell in capabilities ), "related-task must not be recorded as a capability/extension entry", ) # Per-revision overrides are resolved with a shallow per-field merge (rule O2). progress = next( cell for cell in capabilities if cell["surface"] == "notifications/progress" and cell["kind"] == "notification" ) self.assertEqual(self.effective_entry(progress, "2025-11-25")["role"], "both") self.assertEqual(self.effective_entry(progress, "2026-07-28")["role"], "server") roots = next( cell for cell in capabilities if cell["surface"] == "roots" and cell["kind"] == "capability" ) self.assertEqual(self.effective_entry(roots, "2026-07-28")["subfeatures"], []) self.assertEqual(self.effective_entry(roots, "2025-11-25")["subfeatures"], ["listChanged"]) # A `both` row satisfies both sender sides but is never advertisable. tools = next(cell for cell in capabilities if cell["surface"] == "tools" and cell["role"] == "server") self.assertEqual(tools["status"], "supported") self.assertEqual(tools["advertised"], {"tools": {"listChanged": False}}) for cell in capabilities: resolved = [self.effective_entry(cell, revision) for revision in ("2026-07-28", "2025-11-25")] for effective in resolved: if effective["role"] != "server" or effective["kind"] not in {"capability", "extension"}: self.assertIsNone(effective["advertised"], cell["surface"]) if effective["kind"] in {"method", "notification"}: self.assertIsNone(effective["advertised"], cell["surface"]) def test_at_3_1_03_duplicate_surface_identity_is_rejected_by_the_triple(self) -> None: self.write_candidate_inventory("inventory-mutation-duplicate-identity.json") result = self.run_validator(["FR-13"]) combined = result.stdout + result.stderr self.assertNotEqual(result.returncode, 0, result.stdout) self.assertIn("DUPLICATE_SURFACE_IDENTITY", combined) self.assertIn("experimental", combined) def test_at_3_1_03_advertised_fragment_on_a_non_advertisable_role_is_rejected(self) -> None: self.write_candidate_inventory("inventory-mutation-advertised-on-client-role.json") result = self.run_validator(["FR-13"]) combined = result.stdout + result.stderr self.assertNotEqual(result.returncode, 0, result.stdout) self.assertIn("ADVERTISED_ON_NON_ADVERTISABLE", combined) self.assertIn("elicitation", combined) def test_at_3_1_03_supported_server_capability_must_carry_an_advertised_fragment(self) -> None: self.write_candidate_inventory("inventory-mutation-supported-server-without-advertised.json") result = self.run_validator(["FR-13"]) combined = result.stdout + result.stderr self.assertNotEqual(result.returncode, 0, result.stdout) self.assertIn("tools", combined) self.assertIn("advertised", combined.lower()) def test_at_3_1_03_effective_per_era_identities_follow_eligibility_then_o2(self) -> None: """O2 coverage beyond the default 50 identities: effective per-era identity sets. Rule O2 resolves an era in two steps (accepted spec `remaining-ac-spec.md:296-300`): a surface whose `revisions[]` does not contain the era is ABSENT for that era - never `not_applicable` - and only then is the shallow `per_revision[era]` merge applied. The expectations below are the reviewed absence sets read off `surface-inventory.md` sections 3-7, not a copy of the fixture: the applicable set of an era is `REVIEWED_IDENTITY_TRIPLES - REVIEWED_ABSENT_` (35 modern, 43 legacy on this candidate). The 50-identity default union stays a separate ORACLE-INPUT and is never reported as per-era completeness. """ payload = self.load_candidate_fixture() protocol: Any = payload["protocol"] capabilities: Any = protocol["capabilities"] per_era: dict[str, list[tuple[Any, Any, Any]]] = {} progress_reviewed_default = ("server", "notification", "notifications/progress") # The one reviewed per-era identity substitution (surface-inventory.md section 2.1/6): the # progress sender is `server` in the modern revision and `both` in the legacy revision. reviewed_progress_per_era = { "2026-07-28": ("server", "notification", "notifications/progress"), "2025-11-25": ("both", "notification", "notifications/progress"), } for era, absent in ( ("2026-07-28", self.REVIEWED_ABSENT_MODERN), ("2025-11-25", self.REVIEWED_ABSENT_LEGACY), ): with self.subTest(era=era): eligible = [cell for cell in capabilities if era in cell["revisions"]] triples = [ (self.effective_entry(cell, era)["role"], cell["kind"], cell["surface"]) for cell in eligible ] per_era[era] = triples expected = ( self.REVIEWED_IDENTITY_TRIPLES - absent - {progress_reviewed_default} ) | {reviewed_progress_per_era[era]} self.assertEqual( len(triples), len(set(triples)), "an override must not create a duplicate EFFECTIVE identity", ) self.assertEqual( set(triples), expected, f"the applicable identities of {era} must match the reviewed tables", ) self.assertEqual( sorted(absent), sorted( (cell["role"], cell["kind"], cell["surface"]) for cell in capabilities if era not in cell["revisions"] ), "every explicitly absent reviewed identity must be ineligible in that era", ) self.assertEqual( set(triples) & absent, set(), "an absent identity can never be part of the applicable set", ) self.assertEqual(len(per_era["2026-07-28"]), 35) self.assertEqual(len(per_era["2025-11-25"]), 43) for era, triples in per_era.items(): self.assertNotEqual( set(triples), self.REVIEWED_IDENTITY_TRIPLES, f"the default 50-identity union is not the applicable set of {era}", ) # The only identity that resolves to a different ROLE per era is the agreed progress override. overridden = set() for cell in capabilities: if "2026-07-28" in cell["revisions"] and "2025-11-25" in cell["revisions"]: roles = { era: self.effective_entry(cell, era)["role"] for era in ("2026-07-28", "2025-11-25") } if roles["2026-07-28"] != roles["2025-11-25"]: overridden.add((cell["role"], cell["kind"], cell["surface"])) self.assertEqual( overridden, {("server", "notification", "notifications/progress")}, "the agreed per-era role difference is exactly the progress sender override", ) progress = next( cell for cell in capabilities if cell["surface"] == "notifications/progress" and cell["kind"] == "notification" ) self.assertEqual(progress["role"], "server") self.assertEqual(progress["per_revision"]["2025-11-25"]["role"], "both") self.assertEqual(self.effective_entry(progress, "2026-07-28")["role"], "server") self.assertEqual(self.effective_entry(progress, "2025-11-25")["role"], "both") # --- B0 / invalid1 inputs of the agreed Gate table (round-2 corrections) ---------------------- # Both are fixture oracles only: they inspect the input documents and never run the validator, so # nothing here is called validator GREEN. The executable Gate rows live in the `--mode` table above # and stay an honest interface gap until the CLI exposes `--mode`. def test_at_3_1_03_synthetic_no_blocker_variant_is_a_hypothetical_completed_state(self) -> None: """`inventory-synthetic-no-blocker.json` (B0 row) is synthetic, never a status relabel. Every `deferred` reviewed surface is resolved to the hypothetical completed state (`supported`), never to `not_applicable`; advertisable server capability/extension rows carry the exact `advertised` fragment their reviewed `subfeatures` declare; and every flipped row is marked with the owner `atdd-synthetic-no-blocker-input`. The document therefore contains no deferral at all - a no-blocker input under the agreed gate reading and under any stricter one. """ payload = self.load_candidate_fixture("inventory-synthetic-no-blocker.json") protocol: Any = payload["protocol"] capabilities: Any = protocol["capabilities"] reviewed_payload: dict[str, Any] = self.load_candidate_fixture() reviewed: Any = reviewed_payload["protocol"]["capabilities"] reviewed_by_identity = { (cell["role"], cell["kind"], cell["surface"]): cell for cell in reviewed } self.assertEqual( [cell["surface"] for cell in capabilities if cell["status"] == "deferred"], [], "the no-blocker input must not defer any reviewed surface", ) self.assertEqual( sorted(cell["surface"] for cell in capabilities if cell["status"] == "not_applicable"), sorted(cell["surface"] for cell in reviewed if cell["status"] == "not_applicable"), "the reviewed `not_applicable` rows must stay exactly as reviewed", ) flipped = 0 for cell in capabilities: original = reviewed_by_identity[(cell["role"], cell["kind"], cell["surface"])] if original["status"] != "deferred": self.assertEqual( cell, original, f"only deferred rows may differ from the reviewed candidate: {cell['surface']}", ) continue flipped += 1 self.assertEqual(cell["status"], "supported", cell["surface"]) self.assertNotIn("reason", cell, "a hypothetical `supported` row carries no reason") self.assertEqual(cell["owner"], "atdd-synthetic-no-blocker-input", cell["surface"]) self.assertEqual( sorted(cell), sorted(set(original) - {"reason"}), "a flipped row may only change status/reason/advertised/open_subfeatures", ) self.assertEqual(flipped, 26, "every deferred reviewed row must be flipped") for cell in capabilities: if cell["owner"].startswith("atdd-synthetic") and cell["role"] == "server" and cell[ "kind" ] in {"capability", "extension"}: self.assertIsNotNone( cell["advertised"], f"a supported server declaration must advertise: {cell['surface']}", ) def test_at_3_1_03_false_not_applicable_input_keeps_the_planned_reasons(self) -> None: """`inventory-mutation-false-not-applicable.json` (invalid1 row) is the REJECTED predecessor. It is the variant that renames every `deferred` status to `not_applicable` while keeping the planned/deferred reasons, i.e. it claims that applicable, scheduled work is inapplicable. A truthful validity gate must answer `invalid` in BOTH modes and never `complete` / 0 blockers; the executable assertion lives in the `--mode` table, which is an honest interface gap until `--mode` exists. """ payload = self.load_candidate_fixture("inventory-mutation-false-not-applicable.json") protocol: Any = payload["protocol"] capabilities: Any = protocol["capabilities"] reviewed_payload: dict[str, Any] = self.load_candidate_fixture() reviewed: Any = reviewed_payload["protocol"]["capabilities"] reviewed_by_identity = { (cell["role"], cell["kind"], cell["surface"]): cell for cell in reviewed } relabelled = 0 falsely_inapplicable_server_capabilities = [] for cell in capabilities: original = reviewed_by_identity[(cell["role"], cell["kind"], cell["surface"])] if original["status"] != "deferred": self.assertEqual(cell, original, f"only deferred rows may be relabelled: {cell['surface']}") continue relabelled += 1 self.assertEqual(cell["status"], "not_applicable", cell["surface"]) self.assertEqual( cell["reason"], original["reason"], "the relabel keeps the planned/deferred reason - that is the false claim", ) if cell["role"] == "server" and cell["kind"] == "capability": falsely_inapplicable_server_capabilities.append(cell["surface"]) self.assertEqual(relabelled, 26, "every deferred reviewed row must be relabelled") self.assertEqual( sorted(falsely_inapplicable_server_capabilities), ["completions", "logging", "prompts", "tasks"], "applicable server capabilities must never be declared inapplicable", ) if __name__ == "__main__": unittest.main()