feat: harden community production foundation through story 1.5
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import importlib.util
|
||||
import json
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
@@ -38,6 +39,10 @@ class AuthenticatedProductSmokeTests(unittest.TestCase):
|
||||
smoke.agent_mcp_url("https://crank.example.com/", "solo", "health-smoke"),
|
||||
"https://crank.example.com/mcp/v1/solo/health-smoke",
|
||||
)
|
||||
self.assertEqual(
|
||||
smoke.agent_mcp_url("http://127.0.0.1:3302", "solo", "health-smoke", ""),
|
||||
"http://127.0.0.1:3302/v1/solo/health-smoke",
|
||||
)
|
||||
|
||||
def test_resolve_workspace_prefers_authenticated_session_slug(self) -> None:
|
||||
smoke = load_smoke_module()
|
||||
@@ -79,6 +84,116 @@ class AuthenticatedProductSmokeTests(unittest.TestCase):
|
||||
self.assertEqual(payload["params"]["name"], "internal_health_smoke")
|
||||
self.assertEqual(payload["params"]["arguments"], {"probe": "ok"})
|
||||
|
||||
def test_tool_call_requires_non_error_expected_structured_outcome(self) -> None:
|
||||
smoke = load_smoke_module()
|
||||
smoke.validate_tool_call_result({"result": {"isError": False, "structuredContent": {"status": "ok"}}})
|
||||
for value in (
|
||||
{"result": {"isError": True, "structuredContent": {"status": "ok"}}},
|
||||
{"result": {"isError": False, "structuredContent": {"status": "failed"}}},
|
||||
{"error": {"message": "secret-canary"}},
|
||||
[],
|
||||
):
|
||||
with self.subTest(value=value), self.assertRaises(smoke.SmokeError) as raised:
|
||||
smoke.validate_tool_call_result(value)
|
||||
self.assertNotIn("secret-canary", str(raised.exception))
|
||||
|
||||
def test_tools_list_and_key_shape_fail_through_safe_error(self) -> None:
|
||||
smoke = load_smoke_module()
|
||||
for value in (None, {"result": {"tools": ["bad"]}}, {"result": {"tools": [{"name": 7}]}}):
|
||||
with self.subTest(value=value), self.assertRaises(smoke.SmokeError):
|
||||
smoke.validate_tools_list(value, "expected")
|
||||
|
||||
class BadKeyClient:
|
||||
def request_json(self, *args, **kwargs):
|
||||
return smoke.JsonResponse(200, {}, {"api_key": {}})
|
||||
|
||||
with self.assertRaises(smoke.SmokeError):
|
||||
smoke.create_agent_key(BadKeyClient(), "ws", "agent")
|
||||
|
||||
def test_session_termination_uses_delete_without_exposing_key(self) -> None:
|
||||
smoke = load_smoke_module()
|
||||
|
||||
class FakeClient:
|
||||
def request_json(self, method, url, **kwargs):
|
||||
self.call = (method, url, kwargs)
|
||||
return smoke.JsonResponse(204, {}, None)
|
||||
|
||||
client = FakeClient()
|
||||
smoke.terminate_mcp_session(client, "https://private.invalid/mcp", "secret-canary", "session-safe")
|
||||
self.assertEqual(client.call[0], "DELETE")
|
||||
self.assertEqual(client.call[2]["expected"], (204, 404))
|
||||
|
||||
def test_operation_and_agent_versions_come_from_api_responses(self) -> None:
|
||||
smoke = load_smoke_module()
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self):
|
||||
self.requests = []
|
||||
|
||||
def request_json(self, method, path, payload=None, **kwargs):
|
||||
self.requests.append((method, path, payload))
|
||||
if path.endswith("/operations"):
|
||||
return smoke.JsonResponse(200, {}, {"operation_id": "op_safe", "version": 7})
|
||||
if path.endswith("/publish") and "/operations/" in path:
|
||||
return smoke.JsonResponse(200, {}, {"published_version": 7})
|
||||
if path.endswith("/agents"):
|
||||
return smoke.JsonResponse(200, {}, {"agent_id": "agent_safe", "version": 3})
|
||||
if path.endswith("/bindings"):
|
||||
return smoke.JsonResponse(200, {}, {})
|
||||
if path.endswith("/publish") and "/agents/" in path:
|
||||
return smoke.JsonResponse(200, {}, {"published_version": 3})
|
||||
raise AssertionError(path)
|
||||
|
||||
client = FakeClient()
|
||||
operation_id, operation_version = smoke.create_operation(client, "ws", "safe", "http://admin-api:3001")
|
||||
published_operation_version = smoke.publish_operation(client, "ws", operation_id, operation_version)
|
||||
agent_id, agent_version = smoke.create_agent(client, "ws", "safe-agent")
|
||||
published_agent_version = smoke.bind_and_publish_agent(
|
||||
client, "ws", agent_id, agent_version, operation_id, published_operation_version, "safe"
|
||||
)
|
||||
|
||||
self.assertEqual((operation_id, operation_version, published_operation_version), ("op_safe", 7, 7))
|
||||
self.assertEqual((agent_id, agent_version, published_agent_version), ("agent_safe", 3, 3))
|
||||
binding = next(payload for _, path, payload in client.requests if path.endswith("/bindings"))[0]
|
||||
self.assertEqual(binding["operation_version"], 7)
|
||||
|
||||
def test_admin_test_run_uses_created_version_and_rejects_failed_outcome(self) -> None:
|
||||
smoke = load_smoke_module()
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, ok):
|
||||
self.ok = ok
|
||||
self.payload = None
|
||||
|
||||
def request_json(self, method, path, payload=None, **kwargs):
|
||||
self.payload = payload
|
||||
return smoke.JsonResponse(200, {}, {"ok": self.ok, "errors": [{"message": "secret-canary"}]})
|
||||
|
||||
passing = FakeClient(True)
|
||||
smoke.run_operation_test(passing, "ws", "op", 9)
|
||||
self.assertEqual(passing.payload, {"version": 9, "input": {"probe": "ok"}})
|
||||
|
||||
with self.assertRaises(smoke.SmokeError) as raised:
|
||||
smoke.run_operation_test(FakeClient(False), "ws", "op", 9)
|
||||
self.assertNotIn("secret-canary", str(raised.exception))
|
||||
|
||||
def test_http_failure_message_is_bounded_and_redacted(self) -> None:
|
||||
smoke = load_smoke_module()
|
||||
error = smoke.safe_error("http", "unexpected_status", status=500)
|
||||
rendered = str(error)
|
||||
self.assertEqual(rendered, "stage=http code=unexpected_status status=500")
|
||||
self.assertNotIn("http://", rendered)
|
||||
self.assertLessEqual(len(rendered.encode("utf-8")), 256)
|
||||
|
||||
def test_safe_summary_contains_revisions_but_not_secrets_or_payloads(self) -> None:
|
||||
smoke = load_smoke_module()
|
||||
summary = smoke.build_safe_summary("op_safe", 7, "agent_safe", 3)
|
||||
encoded = json.dumps(summary, sort_keys=True)
|
||||
self.assertIn('"operation_version": 7', encoded)
|
||||
self.assertIn('"agent_revision": 3', encoded)
|
||||
for forbidden in ("secret", "cookie", "authorization", "payload", "session"):
|
||||
self.assertNotIn(forbidden, encoded.lower())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,14 +1,26 @@
|
||||
import importlib.util
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
CHECKER = ROOT / "scripts" / "check-community-scope.py"
|
||||
|
||||
|
||||
def load_checker_module():
|
||||
spec = importlib.util.spec_from_file_location("community_scope_checker", CHECKER)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError("cannot load Community scope checker")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class CommunityScopeCheckTests(unittest.TestCase):
|
||||
def run_checker(self, root: Path, files: list[str]) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
@@ -19,6 +31,15 @@ class CommunityScopeCheckTests(unittest.TestCase):
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
def run_default_checker(self, root: Path) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[sys.executable, str(CHECKER), "--root", str(root)],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
def test_passes_for_clean_community_content(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
@@ -91,6 +112,36 @@ class CommunityScopeCheckTests(unittest.TestCase):
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_excluded_explicit_paths_are_validated_before_skip(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "target").mkdir()
|
||||
(root / "target" / "generated.txt").write_text(
|
||||
"generated output\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
valid_excluded = self.run_checker(root, ["target/generated.txt"])
|
||||
self.assertEqual(valid_excluded.returncode, 0, valid_excluded.stderr)
|
||||
|
||||
for raw_path in ["LICENSE", "target/missing.txt", "target/../missing.txt"]:
|
||||
with self.subTest(raw_path=raw_path):
|
||||
rejected = self.run_checker(root, [raw_path])
|
||||
self.assertNotEqual(rejected.returncode, 0)
|
||||
self.assertIn("invalid explicit file path", rejected.stderr)
|
||||
|
||||
def test_default_discovery_ignores_tracked_deletions(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
subprocess.run(["git", "init", "-q"], cwd=root, check=True)
|
||||
deleted = root / "deleted.txt"
|
||||
deleted.write_text("clean\n", encoding="utf-8")
|
||||
subprocess.run(["git", "add", "deleted.txt"], cwd=root, check=True)
|
||||
deleted.unlink()
|
||||
|
||||
result = self.run_default_checker(root)
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_skips_binary_files(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
@@ -100,6 +151,167 @@ class CommunityScopeCheckTests(unittest.TestCase):
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_allows_planned_community_capability_identifiers(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "inventory.txt").write_text(
|
||||
"resources prompts tasks load_runs\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = self.run_checker(root, ["inventory.txt"])
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_rejects_forbidden_identifier_after_narrow_inline_allow(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "scope.txt").write_text(
|
||||
"enterprise_rbac enforcement fixture "
|
||||
"# community-scope: allow=enterprise\n"
|
||||
"enterprise_rbac product capability\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = self.run_checker(root, ["scope.txt"])
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertNotIn("scope.txt:1:", result.stderr)
|
||||
self.assertIn("scope.txt:2: forbidden marker `enterprise`", result.stderr)
|
||||
|
||||
def test_explicit_paths_fail_closed_without_disclosing_host_paths(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
outside = root.parent / "SECRET_CANARY_scope-outside.txt"
|
||||
outside.write_text("clean\n", encoding="utf-8")
|
||||
self.addCleanup(outside.unlink, missing_ok=True)
|
||||
internal_target = root / "target.txt"
|
||||
internal_target.write_text("clean\n", encoding="utf-8")
|
||||
internal_link = root / "internal-link.txt"
|
||||
internal_link.symlink_to(internal_target)
|
||||
outside_link = root / "outside-link.txt"
|
||||
outside_link.symlink_to(outside)
|
||||
|
||||
cases = [
|
||||
"missing.txt",
|
||||
f"../{outside.name}",
|
||||
str(outside),
|
||||
"internal-link.txt",
|
||||
"outside-link.txt",
|
||||
]
|
||||
for raw_path in cases:
|
||||
with self.subTest(raw_path=raw_path):
|
||||
result = self.run_checker(root, [raw_path])
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("invalid explicit file path", result.stderr)
|
||||
self.assertNotIn("SECRET_CANARY", result.stderr)
|
||||
|
||||
def test_invalid_path_report_is_bounded_and_deterministically_truncated(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
missing = [f"missing-{index:04d}.txt" for index in range(1_200, 0, -1)]
|
||||
|
||||
first = self.run_checker(root, missing)
|
||||
second = self.run_checker(root, missing)
|
||||
|
||||
self.assertNotEqual(first.returncode, 0)
|
||||
self.assertEqual(first.stderr, second.stderr)
|
||||
self.assertLessEqual(len(first.stderr.encode("utf-8")), 64 * 1024)
|
||||
self.assertIn("error: findings truncated", first.stderr)
|
||||
self.assertNotIn("missing-", first.stderr)
|
||||
|
||||
def test_single_file_finding_limit_emits_truncation_marker(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "scope.txt").write_text(
|
||||
"grpc\n" * 1_001,
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = self.run_checker(root, ["scope.txt"])
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("error: findings truncated", result.stderr)
|
||||
self.assertLessEqual(len(result.stderr.encode("utf-8")), 64 * 1024)
|
||||
|
||||
def test_exact_finding_limit_is_not_reported_as_truncated(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "scope.txt").write_text("grpc\n" * 1_000, encoding="utf-8")
|
||||
|
||||
result = self.run_checker(root, ["scope.txt"])
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertNotIn("error: findings truncated", result.stderr)
|
||||
|
||||
def test_rejects_camel_and_pascal_case_forbidden_identifiers(self) -> None:
|
||||
cases = {
|
||||
"EnterpriseRbac": "enterprise",
|
||||
"MultiWorkspace": "multi-workspace",
|
||||
"NonRestUpstream": "non-rest-upstream",
|
||||
"ArbitraryDistributedLoadTargets": "distributed-load-targets",
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
for index, (identifier, marker) in enumerate(cases.items()):
|
||||
path = root / f"scope-{index}.txt"
|
||||
path.write_text(identifier + "\n", encoding="utf-8")
|
||||
with self.subTest(identifier=identifier):
|
||||
result = self.run_checker(root, [path.name])
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn(f"forbidden marker `{marker}`", result.stderr)
|
||||
|
||||
def test_invalid_utf8_text_fails_closed(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "scope.txt").write_bytes(b"enterpr\xffise capability\n")
|
||||
|
||||
result = self.run_checker(root, ["scope.txt"])
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("invalid explicit file path", result.stderr)
|
||||
self.assertNotIn("Traceback", result.stderr)
|
||||
|
||||
def test_control_characters_in_explicit_paths_cannot_forge_report_lines(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
forged_path = "forged\nerror: injected.txt"
|
||||
(root / forged_path).write_text("grpc\n", encoding="utf-8")
|
||||
|
||||
result = self.run_checker(root, [forged_path])
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(result.stderr.count("\n"), 2)
|
||||
self.assertNotIn("forged", result.stderr)
|
||||
self.assertNotIn("injected", result.stderr)
|
||||
|
||||
def test_oversized_single_line_fails_closed_without_materializing_it(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "scope.txt").write_text("x" * 65_537, encoding="utf-8")
|
||||
|
||||
result = self.run_checker(root, ["scope.txt"])
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("invalid explicit file path", result.stderr)
|
||||
self.assertNotIn("Traceback", result.stderr)
|
||||
|
||||
def test_open_failure_after_validation_returns_a_safe_failure(self) -> None:
|
||||
checker = load_checker_module()
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "scope.txt").write_text("clean\n", encoding="utf-8")
|
||||
|
||||
with mock.patch.object(Path, "open", side_effect=OSError("host path canary")):
|
||||
findings, valid, truncated, scanned_bytes = checker.scan_file(
|
||||
root, "scope.txt"
|
||||
)
|
||||
|
||||
self.assertEqual(findings, [])
|
||||
self.assertFalse(valid)
|
||||
self.assertFalse(truncated)
|
||||
self.assertEqual(scanned_bytes, 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[2] / "scripts/check-config-boundaries.py"
|
||||
|
||||
|
||||
class ConfigBoundaryTests(unittest.TestCase):
|
||||
def run_check(self, root: Path, *files: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["python3", str(SCRIPT), "--root", str(root), "--files", *files],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def test_value_only_production_code_passes(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
path = root / "crates/example/src/lib.rs"
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text("pub fn configured(value: u32) -> u32 { value }\n", encoding="utf-8")
|
||||
result = self.run_check(root, "crates/example/src/lib.rs")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_direct_alias_and_hidden_reader_forms_fail(self) -> None:
|
||||
cases = (
|
||||
"std::env::var(\"X\")",
|
||||
"std /* split */ ::\n env :: var(\"X\")",
|
||||
"use std::env as process_environment;",
|
||||
"use std::{collections::BTreeMap, env};",
|
||||
"pub use std :: env :: var as exported_reader;",
|
||||
"use std as standard; standard :: env :: var(\"X\");",
|
||||
"extern crate std as standard; standard::env::vars();",
|
||||
"use std::{self as system}; system::env::var(\"X\");",
|
||||
"env::vars_os()",
|
||||
"fn community_from_env() {}",
|
||||
"dotenvy::dotenv()",
|
||||
"option_env!(\"SECRET\")",
|
||||
"env!(\"SECRET\")",
|
||||
)
|
||||
for index, source in enumerate(cases):
|
||||
with self.subTest(source=source), tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
relative = f"crates/example/src/case_{index}.rs"
|
||||
path = root / relative
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(source, encoding="utf-8")
|
||||
result = self.run_check(root, relative)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
|
||||
def test_only_package_version_compile_time_macro_is_allowed(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
relative = "apps/example/src/main.rs"
|
||||
path = root / relative
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(
|
||||
'const VERSION: &str = env! ( "CARGO_PKG_VERSION" );\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = self.run_check(root, relative)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_command_line_arguments_are_not_environment_configuration(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
relative = "apps/example/src/main.rs"
|
||||
path = root / relative
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(
|
||||
"fn main() { let _ = std::env::args().next(); }\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = self.run_check(root, relative)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_comments_and_strings_do_not_trigger_false_positives(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
relative = "crates/example/src/lib.rs"
|
||||
path = root / relative
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(
|
||||
'// std::env::var("X")\nconst NOTE: &str = "option_env!(\\\"X\\\")";\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = self.run_check(root, relative)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_missing_traversal_and_symlink_paths_fail_closed(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
outside = root.parent / "config-boundary-outside.rs"
|
||||
outside.write_text("std::env::var(\"X\");", encoding="utf-8")
|
||||
self.addCleanup(outside.unlink, missing_ok=True)
|
||||
for supplied in ("missing.rs", "../config-boundary-outside.rs"):
|
||||
result = self.run_check(root, supplied)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
link = root / "link.rs"
|
||||
link.symlink_to(outside)
|
||||
result = self.run_check(root, "link.rs")
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
|
||||
def test_default_scan_covers_build_scripts_and_rejects_symlinks(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
build = root / "crates/example/build.rs"
|
||||
build.parent.mkdir(parents=True)
|
||||
build.write_text('std::env::var("X");', encoding="utf-8")
|
||||
result = subprocess.run(
|
||||
["python3", str(SCRIPT), "--root", str(root)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
|
||||
build.unlink()
|
||||
outside = root / "outside.rs"
|
||||
outside.write_text("fn safe() {}", encoding="utf-8")
|
||||
link = root / "apps/example/src/link.rs"
|
||||
link.parent.mkdir(parents=True)
|
||||
link.symlink_to(outside)
|
||||
result = subprocess.run(
|
||||
["python3", str(SCRIPT), "--root", str(root)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,48 @@
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[2] / "scripts" / "check-migration-boundaries.py"
|
||||
|
||||
|
||||
class MigrationBoundaryTests(unittest.TestCase):
|
||||
def run_check(self, root: Path) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["python3", str(SCRIPT), "--root", str(root)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def test_rejects_ddl_runner_and_sql_include_outside_authority(self) -> None:
|
||||
samples = (
|
||||
'sqlx::query("CREATE UNIQUE INDEX idx ON sample(id)");',
|
||||
'sqlx::query("CREATE TEMP TABLE sample(id int)");',
|
||||
'sqlx::query("CREATE TYPE state AS ENUM (\'ready\')");',
|
||||
'sqlx::migrate!("./migrations");',
|
||||
'include_str!("local.sql");',
|
||||
'sqlx::query("INSERT INTO __crank_migrations VALUES (1)");',
|
||||
)
|
||||
for index, sample in enumerate(samples):
|
||||
with self.subTest(index=index), tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
source = root / "apps" / "sample" / "src" / "main.rs"
|
||||
source.parent.mkdir(parents=True)
|
||||
source.write_text(sample, encoding="utf-8")
|
||||
result = self.run_check(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
|
||||
def test_allows_canonical_migration_assets(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
source = root / "crates" / "crank-registry" / "src" / "migrations" / "v1.sql"
|
||||
source.parent.mkdir(parents=True)
|
||||
source.write_text("CREATE TABLE sample(id int);", encoding="utf-8")
|
||||
result = self.run_check(root)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,173 @@
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[2] / "scripts/check-runtime-config.py"
|
||||
|
||||
|
||||
def field(name: str, process: str = "shared", mode: str = "effective") -> dict:
|
||||
return {
|
||||
"env_name": name,
|
||||
"process": process,
|
||||
"mode": mode,
|
||||
"required": False,
|
||||
"default": "value",
|
||||
"sensitivity": "public",
|
||||
}
|
||||
|
||||
|
||||
class RuntimeConfigContractTests(unittest.TestCase):
|
||||
def fixture(self) -> Path:
|
||||
root = Path(tempfile.mkdtemp())
|
||||
(root / "docs/schemas").mkdir(parents=True)
|
||||
(root / "deploy/community").mkdir(parents=True)
|
||||
contract = {
|
||||
"schema_version": 1,
|
||||
"fields": [
|
||||
field("CRANK_SHARED"),
|
||||
field("CRANK_ADMIN", "admin_api"),
|
||||
field("CRANK_MCP", "mcp_server"),
|
||||
field("CRANK_OLD", mode="deprecated_no_effect"),
|
||||
],
|
||||
"deployment_only_fields": ["COMPOSE_PROJECT_NAME"],
|
||||
}
|
||||
(root / "docs/schemas/runtime-config.schema.json").write_text(
|
||||
json.dumps(contract), encoding="utf-8"
|
||||
)
|
||||
section = (
|
||||
"# BEGIN GENERATED CRANK RUNTIME CONFIG\n"
|
||||
"CRANK_ADMIN=value\nCRANK_MCP=value\nCRANK_SHARED=value\n"
|
||||
"# END GENERATED CRANK RUNTIME CONFIG\n"
|
||||
)
|
||||
for relative in (
|
||||
".env.example",
|
||||
"deploy/community/.env.example",
|
||||
"deploy/community/.env.images.example",
|
||||
):
|
||||
(root / relative).write_text(section, encoding="utf-8")
|
||||
compose = (
|
||||
"services:\n"
|
||||
" admin-api:\n environment:\n"
|
||||
" CRANK_SHARED: ${CRANK_SHARED:-value}\n"
|
||||
" CRANK_ADMIN: ${CRANK_ADMIN:-value}\n"
|
||||
" mcp-server:\n environment:\n"
|
||||
" CRANK_SHARED: ${CRANK_SHARED:-value}\n"
|
||||
" CRANK_MCP: ${CRANK_MCP:-value}\n"
|
||||
)
|
||||
for relative in (
|
||||
"docker-compose.yml",
|
||||
"deploy/community/docker-compose.yml",
|
||||
"deploy/community/docker-compose.images.yml",
|
||||
):
|
||||
(root / relative).write_text(compose, encoding="utf-8")
|
||||
return root
|
||||
|
||||
def run_check(self, root: Path) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["python3", str(SCRIPT), "--root", str(root)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def test_valid_contract_passes(self) -> None:
|
||||
result = self.run_check(self.fixture())
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_missing_generated_field_fails_closed(self) -> None:
|
||||
root = self.fixture()
|
||||
path = root / ".env.example"
|
||||
path.write_text(
|
||||
path.read_text().replace("CRANK_MCP=value\n", ""), encoding="utf-8"
|
||||
)
|
||||
result = self.run_check(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
|
||||
def test_compose_extra_or_wrong_process_field_fails(self) -> None:
|
||||
root = self.fixture()
|
||||
path = root / "docker-compose.yml"
|
||||
path.write_text(
|
||||
path.read_text().replace(
|
||||
"CRANK_ADMIN: ${CRANK_ADMIN:-value}",
|
||||
"CRANK_ADMIN: ${CRANK_ADMIN:-value}\n"
|
||||
" CRANK_MCP: ${CRANK_MCP:-value}",
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = self.run_check(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
|
||||
def test_divergent_inline_default_fails(self) -> None:
|
||||
root = self.fixture()
|
||||
path = root / "docker-compose.yml"
|
||||
path.write_text(
|
||||
path.read_text().replace("${CRANK_SHARED:-value}", "${CRANK_SHARED:-drift}", 1),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = self.run_check(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("divergent Compose inline default", result.stderr)
|
||||
|
||||
def test_wrong_interpolation_reference_fails(self) -> None:
|
||||
root = self.fixture()
|
||||
path = root / "docker-compose.yml"
|
||||
path.write_text(
|
||||
path.read_text().replace(
|
||||
"CRANK_ADMIN: ${CRANK_ADMIN:-value}",
|
||||
"CRANK_ADMIN: ${CRANK_SHARED:-value}",
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = self.run_check(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("wrong Compose interpolation reference", result.stderr)
|
||||
|
||||
def test_required_and_empty_operator_forms_are_enforced(self) -> None:
|
||||
for required, expression in (
|
||||
(True, "${CRANK_SHARED:-value}"),
|
||||
(False, "${CRANK_SHARED-value}"),
|
||||
(False, "${CRANK_SHARED:?required}"),
|
||||
):
|
||||
with self.subTest(required=required, expression=expression):
|
||||
root = self.fixture()
|
||||
schema_path = root / "docs/schemas/runtime-config.schema.json"
|
||||
schema = json.loads(schema_path.read_text(encoding="utf-8"))
|
||||
schema["fields"][0]["required"] = required
|
||||
schema_path.write_text(json.dumps(schema), encoding="utf-8")
|
||||
compose_path = root / "docker-compose.yml"
|
||||
compose_path.write_text(
|
||||
compose_path.read_text().replace(
|
||||
"${CRANK_SHARED:-value}", expression, 1
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = self.run_check(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
|
||||
def test_required_empty_aware_error_form_passes(self) -> None:
|
||||
root = self.fixture()
|
||||
schema_path = root / "docs/schemas/runtime-config.schema.json"
|
||||
schema = json.loads(schema_path.read_text(encoding="utf-8"))
|
||||
schema["fields"][0]["required"] = True
|
||||
schema_path.write_text(json.dumps(schema), encoding="utf-8")
|
||||
for relative in (
|
||||
"docker-compose.yml",
|
||||
"deploy/community/docker-compose.yml",
|
||||
"deploy/community/docker-compose.images.yml",
|
||||
):
|
||||
path = root / relative
|
||||
path.write_text(
|
||||
path.read_text().replace(
|
||||
"${CRANK_SHARED:-value}", "${CRANK_SHARED:?must be configured}"
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = self.run_check(root)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -166,6 +166,23 @@ class RustBoundaryCheckTests(unittest.TestCase):
|
||||
self.assertEqual(violations[0].source, "crank-metrics")
|
||||
self.assertEqual(violations[0].dependency, "crank-core")
|
||||
|
||||
def test_rejects_config_contract_dependency_on_workspace_crates(self) -> None:
|
||||
packages = [
|
||||
package(
|
||||
self.root,
|
||||
"crank-config",
|
||||
"crates/crank-config",
|
||||
["crank-core"],
|
||||
),
|
||||
package(self.root, "crank-core", "crates/crank-core"),
|
||||
]
|
||||
|
||||
violations = self.checker.find_violations(metadata(packages, self.root))
|
||||
|
||||
self.assertEqual(len(violations), 1)
|
||||
self.assertEqual(violations[0].source, "crank-config")
|
||||
self.assertEqual(violations[0].dependency, "crank-core")
|
||||
|
||||
def test_rejects_domain_and_runtime_dependencies_on_observability(self) -> None:
|
||||
for source in ("crank-core", "crank-registry", "crank-runtime"):
|
||||
packages = [
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
COLLECTOR = ROOT / "scripts" / "collect-capability-baseline.py"
|
||||
|
||||
|
||||
class CapabilityBaselineCollectorTests(unittest.TestCase):
|
||||
def run_playwright(self, report: object) -> tuple[subprocess.CompletedProcess[str], Path, tempfile.TemporaryDirectory[str]]:
|
||||
temporary = tempfile.TemporaryDirectory()
|
||||
root = Path(temporary.name)
|
||||
report_path = root / "report.json"
|
||||
output_path = root / "candidate.json"
|
||||
report_path.write_text(json.dumps(report), encoding="utf-8")
|
||||
result = subprocess.run(
|
||||
["python3", str(COLLECTOR), "playwright", "--report", str(report_path), "--output", str(output_path), "--source-revision", "0" * 40, "--environment-class", "community-test", "--flow-id", "ui-operations"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
return result, output_path, temporary
|
||||
|
||||
def test_stable_playwright_pass_is_collected(self) -> None:
|
||||
report = {"suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}]}]}]}
|
||||
result, output, temporary = self.run_playwright(report)
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
candidate = json.loads(output.read_text(encoding="utf-8"))
|
||||
self.assertEqual(candidate["execution_verdict"], "pass")
|
||||
self.assertTrue(candidate["accepted"])
|
||||
self.assertEqual(candidate["evidence_mode"], "automated")
|
||||
self.assertRegex(candidate["source_report_sha256"], "^[0-9a-f]{64}$")
|
||||
|
||||
def test_retry_pass_is_flaky_and_not_accepted(self) -> None:
|
||||
report = {"suites": [{"specs": [{"tests": [{"status": "flaky", "results": [{"status": "failed", "retry": 0}, {"status": "passed", "retry": 1}]}]}]}]}
|
||||
result, output, temporary = self.run_playwright(report)
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
candidate = json.loads(output.read_text(encoding="utf-8"))
|
||||
self.assertEqual(candidate["execution_verdict"], "flaky")
|
||||
self.assertFalse(candidate["accepted"])
|
||||
|
||||
def test_skipped_and_missing_results_are_not_pass(self) -> None:
|
||||
report = {"suites": [{"specs": [{"tests": [{"status": "skipped", "results": []}]}]}]}
|
||||
result, output, temporary = self.run_playwright(report)
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(json.loads(output.read_text(encoding="utf-8"))["execution_verdict"], "skipped")
|
||||
|
||||
def test_raw_report_content_never_reaches_candidate_or_error(self) -> None:
|
||||
canary = "Bearer secret-canary /home/private/workspace https://private.invalid?q=secret"
|
||||
report = {"suites": [], "errors": [{"message": canary}], "stdout": [canary]}
|
||||
result, output, temporary = self.run_playwright(report)
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(json.loads(output.read_text(encoding="utf-8"))["execution_verdict"], "fail")
|
||||
self.assertNotIn(canary, output.read_text(encoding="utf-8"))
|
||||
self.assertNotIn(canary, result.stderr)
|
||||
|
||||
def test_report_level_error_and_malformed_result_never_pass(self) -> None:
|
||||
reports = [
|
||||
{"errors": [{"message": "fatal"}], "suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed"}]}]}]}]},
|
||||
{"suites": [{"specs": [{"tests": [{"status": "expected", "results": ["not-an-object"]}]}]}]},
|
||||
]
|
||||
for report in reports:
|
||||
with self.subTest(report=report):
|
||||
result, output, temporary = self.run_playwright(report)
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertFalse(json.loads(output.read_text(encoding="utf-8"))["accepted"])
|
||||
|
||||
def test_oversized_report_fails_without_traceback(self) -> None:
|
||||
temporary = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
root = Path(temporary.name)
|
||||
report = root / "report.json"
|
||||
report.write_bytes(b" " * (4 * 1024 * 1024 + 1))
|
||||
result = subprocess.run(
|
||||
["python3", str(COLLECTOR), "playwright", "--report", str(report), "--output", str(root / "out.json"), "--source-revision", "0" * 40, "--environment-class", "community-test", "--flow-id", "ui-operations"],
|
||||
text=True, capture_output=True, check=False,
|
||||
)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("INPUT_TOO_LARGE", result.stderr)
|
||||
self.assertNotIn("Traceback", result.stderr)
|
||||
|
||||
def test_allowlisted_command_report_derives_exit_verdict(self) -> None:
|
||||
temporary = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
root = Path(temporary.name)
|
||||
report = root / "command.json"
|
||||
output = root / "candidate.json"
|
||||
report.write_text(json.dumps({"command_id": "rust-admin-integration", "exit_code": 7, "timed_out": False, "skipped": 0}), encoding="utf-8")
|
||||
result = subprocess.run(
|
||||
["python3", str(COLLECTOR), "command-report", "--report", str(report), "--output", str(output), "--source-revision", "0" * 40, "--environment-class", "community-test", "--flow-id", "api-operation-test-run"],
|
||||
text=True, capture_output=True, check=False,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
candidate = json.loads(output.read_text(encoding="utf-8"))
|
||||
self.assertEqual(candidate["execution_verdict"], "fail")
|
||||
self.assertFalse(candidate["accepted"])
|
||||
|
||||
def test_unknown_command_report_is_rejected(self) -> None:
|
||||
temporary = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
root = Path(temporary.name)
|
||||
report = root / "command.json"
|
||||
report.write_text(json.dumps({"command_id": "arbitrary-shell", "exit_code": 0, "timed_out": False, "skipped": 0}), encoding="utf-8")
|
||||
result = subprocess.run(
|
||||
["python3", str(COLLECTOR), "command-report", "--report", str(report), "--output", str(root / "candidate.json"), "--source-revision", "0" * 40, "--environment-class", "community-test", "--flow-id", "api-operation-test-run"],
|
||||
text=True, capture_output=True, check=False,
|
||||
)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("UNKNOWN_COMMAND", result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,283 @@
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
VALIDATOR = ROOT / "scripts" / "validate-capability-baseline.py"
|
||||
VERSION = "2026.08.08.1"
|
||||
|
||||
|
||||
def dump(path: Path, value: object) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
class CapabilityBaselineValidatorTests(unittest.TestCase):
|
||||
def make_root(self) -> tuple[tempfile.TemporaryDirectory[str], Path]:
|
||||
temporary = tempfile.TemporaryDirectory()
|
||||
root = Path(temporary.name)
|
||||
inventory = {
|
||||
"schema_version": 1,
|
||||
"product": "crank-community",
|
||||
"flows": [
|
||||
{
|
||||
"id": "api-operation-test-run",
|
||||
"type": "api",
|
||||
"requirements": ["FR-2", "FR-46"],
|
||||
"user_outcome": "An administrator can execute an Operation test run.",
|
||||
"owner": "runtime-community",
|
||||
"status": "implemented",
|
||||
"capabilities": ["tools"],
|
||||
"evidence": {"automated": ["docs/capability-baseline/results.json"], "manual": ["docs/capability-baseline/manual-checklist.md"]},
|
||||
}
|
||||
],
|
||||
}
|
||||
required = {
|
||||
"baseline_version": VERSION,
|
||||
"required_flow_ids": ["api-operation-test-run"],
|
||||
"surface_groups": [{"id": "operations", "flow_ids": ["api-operation-test-run"]}],
|
||||
}
|
||||
taxonomy = {
|
||||
"baseline_version": VERSION,
|
||||
"implementation_statuses": ["implemented", "planned", "gap", "blocked"],
|
||||
"execution_verdicts": ["pass", "fail", "blocked", "skipped", "flaky", "not_run"],
|
||||
"evidence_modes": ["automated", "manual_only"],
|
||||
"full_pass": {"implementation_status": "implemented", "execution_verdict": "pass", "evidence_mode": "automated"},
|
||||
}
|
||||
checklist = f"# Community UI baseline checklist\n\nbaseline_version: {VERSION}\n\n## UI-01 Operations\n\n- flow_id: api-operation-test-run\n- states: happy, error, recovery\n- verdict: not_run\n- reason: API-only fixture\n"
|
||||
results = {
|
||||
"baseline_version": VERSION,
|
||||
"source_revision": "0123456789abcdef0123456789abcdef01234567",
|
||||
"environment_class": "community-test",
|
||||
"runs": [{
|
||||
"id": "run-api-operation-test",
|
||||
"command_id": "python-tooling-tests",
|
||||
"flow_ids": ["api-operation-test-run"],
|
||||
"execution_verdict": "pass",
|
||||
"evidence_mode": "automated",
|
||||
"accepted": True,
|
||||
"source_report_sha256": "a" * 64,
|
||||
"collector": "capability-baseline-collector-v1",
|
||||
"source_revision": "0123456789abcdef0123456789abcdef01234567",
|
||||
"environment_class": "community-test",
|
||||
}],
|
||||
"manual_results": [{
|
||||
"check_id": "UI-01",
|
||||
"evidence_mode": "manual_only",
|
||||
"execution_verdict": "not_run",
|
||||
"flow_ids": ["api-operation-test-run"],
|
||||
"next_evidence": "Automate this fixture state.",
|
||||
}],
|
||||
"defects": [],
|
||||
}
|
||||
paths = {
|
||||
"inventory": root / "docs/capability-inventory.json",
|
||||
"required_surfaces": root / "docs/capability-baseline/required-surfaces.json",
|
||||
"taxonomy": root / "docs/capability-baseline/outcome-taxonomy.json",
|
||||
"checklist": root / "docs/capability-baseline/manual-checklist.md",
|
||||
"results": root / "docs/capability-baseline/results.json",
|
||||
}
|
||||
dump(paths["inventory"], inventory)
|
||||
dump(paths["required_surfaces"], required)
|
||||
dump(paths["taxonomy"], taxonomy)
|
||||
paths["checklist"].write_text(checklist, encoding="utf-8")
|
||||
dump(paths["results"], results)
|
||||
schema = {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://crank.local/schemas/capability-baseline.schema.json",
|
||||
"title": "Crank Community Capability Baseline Manifest",
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": ["schema_version", "baseline_version", "artifacts"],
|
||||
"properties": {
|
||||
"schema_version": {"const": 1},
|
||||
"baseline_version": {"type": "string", "pattern": "^[0-9]{4}\\.[0-9]{2}\\.[0-9]{2}\\.[1-9][0-9]*$"},
|
||||
"artifacts": {"type": "array", "minItems": 5, "maxItems": 5},
|
||||
},
|
||||
"$defs": {
|
||||
"artifact": {"type": "object", "additionalProperties": False},
|
||||
"taxonomy": {
|
||||
"type": "object", "additionalProperties": False,
|
||||
"properties": {"full_pass": {"properties": {"implementation_status": {}, "execution_verdict": {}, "evidence_mode": {}}}},
|
||||
},
|
||||
"run": {
|
||||
"type": "object", "additionalProperties": False,
|
||||
"required": ["id", "command_id", "flow_ids", "execution_verdict", "evidence_mode", "accepted", "source_report_sha256", "source_revision", "environment_class", "collector"],
|
||||
},
|
||||
"defect": {
|
||||
"type": "object", "additionalProperties": False,
|
||||
"required": ["id", "severity", "steps", "contract", "owner", "flow_ids", "next_action"],
|
||||
},
|
||||
"manual_result": {"type": "object", "additionalProperties": False},
|
||||
"results": {"type": "object", "additionalProperties": False},
|
||||
},
|
||||
}
|
||||
dump(root / "docs/schemas/capability-baseline.schema.json", schema)
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"baseline_version": VERSION,
|
||||
"artifacts": [
|
||||
{"kind": kind, "path": str(path.relative_to(root)), "sha256": sha256(path)}
|
||||
for kind, path in paths.items()
|
||||
],
|
||||
}
|
||||
dump(root / "docs/capability-baseline/manifest.json", manifest)
|
||||
subprocess.run(["git", "init", "-q", str(root)], check=True)
|
||||
subprocess.run(
|
||||
["git", "-C", str(root), "add", *[str(path.relative_to(root)) for path in paths.values()]],
|
||||
check=True,
|
||||
)
|
||||
return temporary, root
|
||||
|
||||
def run_validator(self, root: Path) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["python3", str(VALIDATOR), "--root", str(root), "--manifest", "docs/capability-baseline/manifest.json", "--schema", "docs/schemas/capability-baseline.schema.json"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def mutate_results(self, root: Path, mutate) -> None:
|
||||
path = root / "docs/capability-baseline/results.json"
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
mutate(value)
|
||||
dump(path, value)
|
||||
manifest_path = root / "docs/capability-baseline/manifest.json"
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
next(item for item in manifest["artifacts"] if item["kind"] == "results")["sha256"] = sha256(path)
|
||||
dump(manifest_path, manifest)
|
||||
|
||||
def mutate_artifact(self, root: Path, kind: str, mutate) -> None:
|
||||
manifest_path = root / "docs/capability-baseline/manifest.json"
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
item = next(entry for entry in manifest["artifacts"] if entry["kind"] == kind)
|
||||
path = root / item["path"]
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
mutate(value)
|
||||
dump(path, value)
|
||||
item["sha256"] = sha256(path)
|
||||
dump(manifest_path, manifest)
|
||||
|
||||
def test_valid_snapshot_passes(self) -> None:
|
||||
temporary, root = self.make_root()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
result = self.run_validator(root)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("baseline_version=2026.08.08.1", result.stdout)
|
||||
|
||||
def test_checksum_drift_fails_closed(self) -> None:
|
||||
temporary, root = self.make_root()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
with (root / "docs/capability-baseline/manual-checklist.md").open("a", encoding="utf-8") as file:
|
||||
file.write("\nchanged\n")
|
||||
result = self.run_validator(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("CHECKSUM_MISMATCH", result.stderr)
|
||||
|
||||
def test_unknown_flow_and_non_pass_cannot_be_accepted(self) -> None:
|
||||
temporary, root = self.make_root()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.mutate_results(root, lambda value: value["runs"][0].update(flow_ids=["missing-flow"], execution_verdict="flaky"))
|
||||
result = self.run_validator(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("UNKNOWN_FLOW_ID", result.stderr)
|
||||
self.assertIn("NON_PASS_RECORDED_AS_PASS", result.stderr)
|
||||
|
||||
def test_severe_defect_requires_blocked_inventory_flow(self) -> None:
|
||||
temporary, root = self.make_root()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.mutate_results(root, lambda value: value["defects"].append({
|
||||
"id": "DEF-001", "severity": "High", "flow_ids": ["api-operation-test-run"],
|
||||
"contract": "Admin test run", "owner": "runtime-community", "steps": ["Run bounded fixture"], "next_action": "Fix downstream",
|
||||
}))
|
||||
result = self.run_validator(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("SEVERE_DEFECT_FLOW_NOT_BLOCKED", result.stderr)
|
||||
|
||||
def test_version_mismatch_and_broken_artifact_path_are_safe(self) -> None:
|
||||
temporary, root = self.make_root()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
manifest_path = root / "docs/capability-baseline/manifest.json"
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
manifest["baseline_version"] = "2026.08.08.2"
|
||||
manifest["artifacts"][0]["path"] = "../outside-secret.txt"
|
||||
dump(manifest_path, manifest)
|
||||
result = self.run_validator(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("BROKEN_ARTIFACT_LINK", result.stderr)
|
||||
self.assertNotIn(str(root), result.stderr)
|
||||
|
||||
def test_unsupported_schema_applicators_fail_closed_at_any_depth(self) -> None:
|
||||
for mutation in (
|
||||
lambda schema: schema.update({"not": {}}),
|
||||
lambda schema: schema["properties"]["artifacts"].update({"allOf": []}),
|
||||
):
|
||||
with self.subTest(mutation=mutation):
|
||||
temporary, root = self.make_root()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
schema_path = root / "docs/schemas/capability-baseline.schema.json"
|
||||
schema = json.loads(schema_path.read_text(encoding="utf-8"))
|
||||
mutation(schema)
|
||||
dump(schema_path, schema)
|
||||
result = self.run_validator(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("INVALID_SCHEMA_CONTRACT", result.stderr)
|
||||
|
||||
def test_required_surfaces_cannot_omit_or_duplicate_implemented_flow(self) -> None:
|
||||
temporary, root = self.make_root()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.mutate_artifact(root, "required_surfaces", lambda value: value.update(required_flow_ids=[], surface_groups=[]))
|
||||
result = self.run_validator(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("INVALID_REQUIRED_SURFACES", result.stderr)
|
||||
|
||||
def test_manual_results_and_taxonomy_truth_table_are_enforced(self) -> None:
|
||||
temporary, root = self.make_root()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.mutate_results(root, lambda value: value["manual_results"][0].pop("next_evidence"))
|
||||
self.mutate_artifact(root, "taxonomy", lambda value: value["full_pass"].update(implementation_status="planned"))
|
||||
result = self.run_validator(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("INVALID_MANUAL_EVIDENCE", result.stderr)
|
||||
self.assertIn("/taxonomy/full_pass", result.stderr)
|
||||
|
||||
def test_accepted_run_requires_known_command_matching_provenance_and_implemented_flow(self) -> None:
|
||||
temporary, root = self.make_root()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.mutate_results(root, lambda value: value["runs"][0].update(command_id="fabricated", source_revision="f" * 40, flow_ids=[{}]))
|
||||
result = self.run_validator(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("UNKNOWN_COMMAND", result.stderr)
|
||||
self.assertIn("PROVENANCE_MISMATCH", result.stderr)
|
||||
self.assertNotIn("Traceback", result.stderr)
|
||||
|
||||
def test_planned_flow_cannot_receive_accepted_evidence(self) -> None:
|
||||
temporary, root = self.make_root()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.mutate_artifact(root, "inventory", lambda value: value["flows"].append({
|
||||
**value["flows"][0], "id": "planned-resource-read", "status": "planned",
|
||||
}))
|
||||
self.mutate_results(root, lambda value: value["runs"][0].update(flow_ids=["planned-resource-read"]))
|
||||
result = self.run_validator(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("NON_PASS_RECORDED_AS_PASS", result.stderr)
|
||||
|
||||
def test_untracked_canonical_artifact_fails_closed(self) -> None:
|
||||
temporary, root = self.make_root()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
subprocess.run(["git", "-C", str(root), "rm", "--cached", "docs/capability-baseline/results.json"], check=True, capture_output=True)
|
||||
result = self.run_validator(root)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("UNTRACKED_ARTIFACT", result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,842 @@
|
||||
import copy
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
VALIDATOR = ROOT / "scripts" / "validate-capability-inventory.py"
|
||||
CANONICAL_INVENTORY = ROOT / "docs" / "capability-inventory.json"
|
||||
CANONICAL_SCHEMA = ROOT / "docs" / "schemas" / "capability-inventory.schema.json"
|
||||
COMMUNITY_CHECKER = ROOT / "scripts" / "check-community-scope.py"
|
||||
COMMUNITY_README = ROOT / "README.md"
|
||||
DOCS_INDEX = ROOT / "docs" / "README.md"
|
||||
COMMUNITY_INTRO = ROOT / "docs" / "intro.md"
|
||||
COMMUNITY_README_EN = ROOT / "docs" / "en" / "README.md"
|
||||
JUSTFILE = ROOT / "justfile"
|
||||
CI_WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
|
||||
RELEASE_WORKFLOW = ROOT / ".gitea" / "workflows" / "release.yml"
|
||||
REQUIRED_SURFACES = ROOT / "docs" / "capability-baseline" / "required-surfaces.json"
|
||||
|
||||
|
||||
def load_validator_module():
|
||||
spec = importlib.util.spec_from_file_location("capability_inventory_validator", VALIDATOR)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError("cannot load capability inventory validator")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
class CapabilityInventoryValidatorTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._temporary_directory = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self._temporary_directory.name)
|
||||
self.inventory_path = self.root / "docs" / "capability-inventory.json"
|
||||
self.schema_path = (
|
||||
self.root / "docs" / "schemas" / "capability-inventory.schema.json"
|
||||
)
|
||||
self.inventory_path.parent.mkdir(parents=True)
|
||||
self.schema_path.parent.mkdir(parents=True)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._temporary_directory.cleanup()
|
||||
|
||||
def write_evidence(self, relative_path: str, content: str = "verified\n") -> str:
|
||||
path = self.root / relative_path
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
return relative_path
|
||||
|
||||
def valid_flow(
|
||||
self,
|
||||
flow_id: str,
|
||||
flow_type: str,
|
||||
requirement: str,
|
||||
status: str = "implemented",
|
||||
capabilities: list[str] | None = None,
|
||||
) -> dict[str, object]:
|
||||
automated = self.write_evidence(
|
||||
f"evidence/automated/{flow_id}.txt",
|
||||
f"{flow_id}: automated acceptance evidence\n",
|
||||
)
|
||||
manual = self.write_evidence(
|
||||
f"evidence/manual/{flow_id}.md",
|
||||
f"# {flow_id}\n\nManual verification complete.\n",
|
||||
)
|
||||
return {
|
||||
"id": flow_id,
|
||||
"type": flow_type,
|
||||
"requirements": [requirement],
|
||||
"user_outcome": f"User completes {flow_id.replace('-', ' ')}.",
|
||||
"owner": "community-release",
|
||||
"status": status,
|
||||
"capabilities": capabilities or ["tools"],
|
||||
"evidence": {
|
||||
"automated": [automated],
|
||||
"manual": [manual],
|
||||
},
|
||||
}
|
||||
|
||||
def write_inventory(self, flows: list[dict[str, object]]) -> None:
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"product": "crank-community",
|
||||
"flows": flows,
|
||||
}
|
||||
self.inventory_path.write_text(
|
||||
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def install_canonical_schema(self) -> None:
|
||||
self.assertTrue(
|
||||
CANONICAL_SCHEMA.is_file(),
|
||||
f"Story 1.1 must create {CANONICAL_SCHEMA.relative_to(ROOT)}",
|
||||
)
|
||||
shutil.copyfile(CANONICAL_SCHEMA, self.schema_path)
|
||||
|
||||
def run_validator(
|
||||
self,
|
||||
required_frs: list[str],
|
||||
install_schema: bool = True,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
if install_schema:
|
||||
self.install_canonical_schema()
|
||||
command = [
|
||||
sys.executable,
|
||||
str(VALIDATOR),
|
||||
"--root",
|
||||
str(self.root),
|
||||
"--inventory",
|
||||
str(self.inventory_path.relative_to(self.root)),
|
||||
"--schema",
|
||||
str(self.schema_path.relative_to(self.root)),
|
||||
]
|
||||
for requirement in required_frs:
|
||||
command.extend(["--required-fr", requirement])
|
||||
return subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
def test_accepts_complete_inventory_with_all_flow_types_and_statuses(self) -> None:
|
||||
flows = [
|
||||
self.valid_flow("ui-create-method", "ui", "FR-1", "implemented"),
|
||||
self.valid_flow("api-create-method", "api", "FR-2", "planned"),
|
||||
self.valid_flow("mcp-tool-call", "mcp", "FR-3", "gap"),
|
||||
self.valid_flow("mcp-resource-read", "mcp", "FR-4", "blocked"),
|
||||
]
|
||||
self.write_inventory(flows)
|
||||
|
||||
result = self.run_validator(["FR-1", "FR-2", "FR-3", "FR-4"])
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("Capability inventory validation passed", result.stdout)
|
||||
self.assertRegex(result.stdout, r"\btotal=4\b")
|
||||
self.assertRegex(result.stdout, r"\bimplemented=1\b")
|
||||
self.assertRegex(result.stdout, r"\bplanned=1\b")
|
||||
self.assertRegex(result.stdout, r"\bgap=1\b")
|
||||
self.assertRegex(result.stdout, r"\bblocked=1\b")
|
||||
|
||||
def test_counts_only_implemented_flows_as_pass(self) -> None:
|
||||
flows = [
|
||||
self.valid_flow("implemented-flow", "api", "FR-1", "implemented"),
|
||||
self.valid_flow("planned-flow", "ui", "FR-2", "planned"),
|
||||
self.valid_flow("gap-flow", "mcp", "FR-3", "gap"),
|
||||
self.valid_flow("blocked-flow", "mcp", "FR-4", "blocked"),
|
||||
]
|
||||
self.write_inventory(flows)
|
||||
|
||||
result = self.run_validator(["FR-1", "FR-2", "FR-3", "FR-4"])
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertRegex(result.stdout, r"\bpass=1\b")
|
||||
self.assertNotRegex(result.stdout, r"\bpass=4\b")
|
||||
|
||||
def test_rejects_duplicate_id_unknown_enum_and_missing_required_fields(self) -> None:
|
||||
base = self.valid_flow("flow-a", "api", "FR-1")
|
||||
cases: list[tuple[str, list[dict[str, object]], str]] = []
|
||||
|
||||
duplicate = copy.deepcopy(base)
|
||||
duplicate["user_outcome"] = "A distinct outcome with a duplicate identifier."
|
||||
cases.append(("duplicate ID", [base, duplicate], "DUPLICATE_FLOW_ID"))
|
||||
|
||||
unknown_status = copy.deepcopy(base)
|
||||
unknown_status["status"] = "done"
|
||||
cases.append(("unknown status", [unknown_status], "UNKNOWN_STATUS"))
|
||||
|
||||
unknown_type = copy.deepcopy(base)
|
||||
unknown_type["type"] = "worker"
|
||||
cases.append(("unknown type", [unknown_type], "UNKNOWN_FLOW_TYPE"))
|
||||
|
||||
no_owner = copy.deepcopy(base)
|
||||
no_owner.pop("owner")
|
||||
cases.append(("missing owner", [no_owner], "MISSING_OWNER"))
|
||||
|
||||
no_outcome = copy.deepcopy(base)
|
||||
no_outcome.pop("user_outcome")
|
||||
cases.append(("missing required field", [no_outcome], "MISSING_REQUIRED_FIELD"))
|
||||
|
||||
for label, flows, expected_code in cases:
|
||||
with self.subTest(label=label):
|
||||
self.write_inventory(flows)
|
||||
result = self.run_validator(["FR-1"])
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn(f"error[{expected_code}]", result.stderr)
|
||||
|
||||
def test_rejects_missing_mandatory_requirement(self) -> None:
|
||||
self.write_inventory([self.valid_flow("only-fr-one", "api", "FR-1")])
|
||||
|
||||
result = self.run_validator(["FR-1", "FR-46"])
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("error[MISSING_REQUIRED_FR]", result.stderr)
|
||||
self.assertIn("FR-46", result.stderr)
|
||||
|
||||
def test_rejects_missing_non_regular_and_escaping_evidence_links(self) -> None:
|
||||
base = self.valid_flow("evidence-flow", "mcp", "FR-46")
|
||||
outside = self.root.parent / f"{self.root.name}-outside.txt"
|
||||
outside.write_text("outside\n", encoding="utf-8")
|
||||
self.addCleanup(outside.unlink, missing_ok=True)
|
||||
internal_target = self.write_evidence("evidence/automated/internal-target.txt")
|
||||
internal_link = self.root / "evidence" / "automated" / "internal-link.txt"
|
||||
internal_link.symlink_to(self.root / internal_target)
|
||||
outside_link = self.root / "evidence" / "automated" / "outside-link.txt"
|
||||
outside_link.symlink_to(outside)
|
||||
broken_cases = [
|
||||
"evidence/automated/does-not-exist.txt",
|
||||
"evidence/automated",
|
||||
f"../{outside.name}",
|
||||
str(outside),
|
||||
"evidence/automated/internal-link.txt",
|
||||
"evidence/automated/outside-link.txt",
|
||||
]
|
||||
|
||||
for evidence_path in broken_cases:
|
||||
with self.subTest(evidence_path=evidence_path):
|
||||
flow = copy.deepcopy(base)
|
||||
flow["evidence"] = {
|
||||
"automated": [evidence_path],
|
||||
"manual": ["evidence/manual/evidence-flow.md"],
|
||||
}
|
||||
self.write_inventory([flow])
|
||||
result = self.run_validator(["FR-46"])
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("error[BROKEN_EVIDENCE_LINK]", result.stderr)
|
||||
|
||||
def test_report_is_deterministic_bounded_sorted_and_redacted(self) -> None:
|
||||
secret_canary = "SECRET_CANARY_sk-live-4hQ9Vw8x"
|
||||
flows = []
|
||||
for index in range(1_200, 0, -1):
|
||||
flow = self.valid_flow(
|
||||
f"invalid-flow-{index:04d}",
|
||||
"api",
|
||||
f"FR-{index}",
|
||||
)
|
||||
flow["status"] = "unknown"
|
||||
flow["user_outcome"] = f"{secret_canary}:{index}"
|
||||
flows.append(flow)
|
||||
self.write_inventory(flows)
|
||||
raw_inventory = self.inventory_path.read_text(encoding="utf-8")
|
||||
|
||||
first = self.run_validator(["FR-46"])
|
||||
second = self.run_validator(["FR-46"])
|
||||
|
||||
self.assertNotEqual(first.returncode, 0)
|
||||
self.assertEqual(first.stderr, second.stderr)
|
||||
self.assertLessEqual(len(first.stderr.encode("utf-8")), 64 * 1024)
|
||||
self.assertNotIn(secret_canary, first.stderr)
|
||||
self.assertNotIn(raw_inventory, first.stderr)
|
||||
diagnostics = re.findall(r"error\[([^]]+)\] ([^:]+):", first.stderr)
|
||||
self.assertTrue(diagnostics)
|
||||
self.assertEqual(diagnostics[:-1], sorted(diagnostics[:-1]))
|
||||
self.assertEqual(diagnostics[-1][0], "TRUNCATED")
|
||||
self.assertNotRegex(first.stderr, r"invalid-flow-\d{4}")
|
||||
|
||||
def test_rejects_malformed_documents_and_schema_contract_drift(self) -> None:
|
||||
self.write_inventory([self.valid_flow("valid-flow", "api", "FR-46")])
|
||||
|
||||
self.inventory_path.write_text("{not-json", encoding="utf-8")
|
||||
malformed_inventory = self.run_validator(["FR-46"])
|
||||
self.assertNotEqual(malformed_inventory.returncode, 0)
|
||||
self.assertIn("error[INVALID_JSON]", malformed_inventory.stderr)
|
||||
|
||||
self.write_inventory([self.valid_flow("valid-flow", "api", "FR-46")])
|
||||
self.schema_path.write_text("{not-json", encoding="utf-8")
|
||||
malformed_schema = self.run_validator(["FR-46"], install_schema=False)
|
||||
self.assertNotEqual(malformed_schema.returncode, 0)
|
||||
self.assertIn("error[INVALID_SCHEMA]", malformed_schema.stderr)
|
||||
|
||||
self.install_canonical_schema()
|
||||
schema = json.loads(self.schema_path.read_text(encoding="utf-8"))
|
||||
schema["$schema"] = "https://json-schema.org/draft/2019-09/schema"
|
||||
self.schema_path.write_text(json.dumps(schema), encoding="utf-8")
|
||||
drifted_schema = self.run_validator(["FR-46"], install_schema=False)
|
||||
self.assertNotEqual(drifted_schema.returncode, 0)
|
||||
self.assertIn("error[INVALID_SCHEMA_CONTRACT]", drifted_schema.stderr)
|
||||
|
||||
def test_rejects_duplicate_nonstandard_recursive_and_large_integer_json(self) -> None:
|
||||
malformed_documents = {
|
||||
"duplicate member": (
|
||||
'{"schema_version":1,"schema_version":1,'
|
||||
'"product":"crank-community","flows":[]}',
|
||||
"INVALID_JSON",
|
||||
),
|
||||
"non-standard number": (
|
||||
'{"schema_version":1,"product":NaN,"flows":[]}',
|
||||
"INVALID_JSON",
|
||||
),
|
||||
"recursive": ("[" * 100_000 + "0" + "]" * 100_000, "INVALID_JSON"),
|
||||
"large integer": ("9" * 5_000, "INVALID_JSON"),
|
||||
}
|
||||
|
||||
for label, (content, expected_code) in malformed_documents.items():
|
||||
with self.subTest(label=label):
|
||||
self.inventory_path.write_text(content, encoding="utf-8")
|
||||
result = self.run_validator(["FR-46"])
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn(f"error[{expected_code}]", result.stderr)
|
||||
self.assertNotIn("Traceback", result.stderr)
|
||||
self.assertLessEqual(len(result.stderr.encode("utf-8")), 64 * 1024)
|
||||
|
||||
def test_schema_parity_rejects_required_type_ref_and_pattern_drift(self) -> None:
|
||||
self.write_inventory([self.valid_flow("schema-parity", "api", "FR-46")])
|
||||
mutations = {
|
||||
"root required missing": lambda schema: schema.pop("required"),
|
||||
"root contract inversion": lambda schema: schema.update({"not": {}}),
|
||||
"malformed evidence required": lambda schema: schema["$defs"]["evidence"].update(
|
||||
{"required": [{}]}
|
||||
),
|
||||
"flow ref drift": lambda schema: schema["properties"]["flows"]["items"].update(
|
||||
{"$ref": "#/$defs/evidence"}
|
||||
),
|
||||
"owner type drift": lambda schema: schema["$defs"]["flow"]["properties"][
|
||||
"owner"
|
||||
].update({"type": "integer"}),
|
||||
"owner const drift": lambda schema: schema["$defs"]["flow"]["properties"][
|
||||
"owner"
|
||||
].update({"const": "impossible"}),
|
||||
"nested applicator drift": lambda schema: schema["$defs"]["evidencePaths"][
|
||||
"items"
|
||||
].update({"anyOf": [{"type": "integer"}]}),
|
||||
"evidence pattern drift": lambda schema: schema["$defs"]["evidencePaths"][
|
||||
"items"
|
||||
].update({"pattern": ".+"}),
|
||||
}
|
||||
|
||||
for label, mutate in mutations.items():
|
||||
with self.subTest(label=label):
|
||||
self.install_canonical_schema()
|
||||
schema = json.loads(self.schema_path.read_text(encoding="utf-8"))
|
||||
mutate(schema)
|
||||
self.schema_path.write_text(json.dumps(schema), encoding="utf-8")
|
||||
result = self.run_validator(["FR-46"], install_schema=False)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("error[INVALID_SCHEMA_CONTRACT]", result.stderr)
|
||||
self.assertNotIn("Traceback", result.stderr)
|
||||
|
||||
self.install_canonical_schema()
|
||||
schema = json.loads(self.schema_path.read_text(encoding="utf-8"))
|
||||
schema["$defs"]["flow"]["properties"]["status"]["enum"].reverse()
|
||||
self.schema_path.write_text(json.dumps(schema), encoding="utf-8")
|
||||
reordered_enum = self.run_validator(["FR-46"], install_schema=False)
|
||||
self.assertEqual(reordered_enum.returncode, 0, reordered_enum.stderr)
|
||||
|
||||
def test_rejects_boolean_schema_version_and_whitespace_required_text(self) -> None:
|
||||
base = self.valid_flow("strict-scalars", "api", "FR-46")
|
||||
payload_cases: list[tuple[str, dict[str, object], str]] = []
|
||||
|
||||
boolean_version = {
|
||||
"schema_version": True,
|
||||
"product": "crank-community",
|
||||
"flows": [copy.deepcopy(base)],
|
||||
}
|
||||
payload_cases.append(("boolean inventory version", boolean_version, "MISSING_REQUIRED_FIELD"))
|
||||
|
||||
whitespace_owner = copy.deepcopy(base)
|
||||
whitespace_owner["owner"] = " \t\n "
|
||||
payload_cases.append(
|
||||
(
|
||||
"whitespace owner",
|
||||
{"schema_version": 1, "product": "crank-community", "flows": [whitespace_owner]},
|
||||
"MISSING_OWNER",
|
||||
)
|
||||
)
|
||||
|
||||
whitespace_outcome = copy.deepcopy(base)
|
||||
whitespace_outcome["user_outcome"] = " \t\n "
|
||||
payload_cases.append(
|
||||
(
|
||||
"whitespace outcome",
|
||||
{"schema_version": 1, "product": "crank-community", "flows": [whitespace_outcome]},
|
||||
"LIMIT_EXCEEDED",
|
||||
)
|
||||
)
|
||||
|
||||
for label, payload, expected_code in payload_cases:
|
||||
with self.subTest(label=label):
|
||||
self.inventory_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
result = self.run_validator(["FR-46"])
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn(f"error[{expected_code}]", result.stderr)
|
||||
|
||||
self.write_inventory([base])
|
||||
self.install_canonical_schema()
|
||||
schema = json.loads(self.schema_path.read_text(encoding="utf-8"))
|
||||
schema["properties"]["schema_version"]["const"] = True
|
||||
self.schema_path.write_text(json.dumps(schema), encoding="utf-8")
|
||||
schema_result = self.run_validator(["FR-46"], install_schema=False)
|
||||
self.assertNotEqual(schema_result.returncode, 0)
|
||||
self.assertIn("error[INVALID_SCHEMA_CONTRACT]", schema_result.stderr)
|
||||
|
||||
def test_diagnostic_accumulation_is_capped_counted_and_redacted(self) -> None:
|
||||
secret_canary = "SECRET_CANARY_unknown-field"
|
||||
flow = self.valid_flow("bounded-diagnostics", "api", "FR-46")
|
||||
for index in range(2_000):
|
||||
flow[f"{secret_canary}-{index:04d}"] = True
|
||||
self.write_inventory([flow])
|
||||
|
||||
result = self.run_validator(["FR-46"])
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("error[TRUNCATED] /: diagnostics omitted=1000", result.stderr)
|
||||
self.assertEqual(result.stderr.count("error[UNKNOWN_FIELD]"), 1_000)
|
||||
self.assertNotIn(secret_canary, result.stderr)
|
||||
self.assertLessEqual(len(result.stderr.encode("utf-8")), 64 * 1024)
|
||||
|
||||
def test_rejects_oversized_document_and_field_limits(self) -> None:
|
||||
self.inventory_path.write_bytes(b" " * (4_194_304 + 1))
|
||||
oversized = self.run_validator(["FR-46"])
|
||||
self.assertNotEqual(oversized.returncode, 0)
|
||||
self.assertIn("error[INPUT_TOO_LARGE]", oversized.stderr)
|
||||
|
||||
flow = self.valid_flow("bounded-flow", "api", "FR-46")
|
||||
flow["user_outcome"] = "x" * 4_097
|
||||
self.write_inventory([flow])
|
||||
overlong = self.run_validator(["FR-46"])
|
||||
self.assertNotEqual(overlong.returncode, 0)
|
||||
self.assertIn("error[LIMIT_EXCEEDED]", overlong.stderr)
|
||||
|
||||
def test_named_string_limits_cover_below_at_and_above_boundaries(self) -> None:
|
||||
string_cases = [
|
||||
("id", 128, lambda length: "a" * length),
|
||||
("owner", 256, lambda length: "o" * length),
|
||||
("user_outcome", 4_096, lambda length: "u" * length),
|
||||
("notes", 4_096, lambda length: "n" * length),
|
||||
("requirements", 128, lambda length: "FR-" + "1" * (length - 3)),
|
||||
]
|
||||
|
||||
for field, limit, make_value in string_cases:
|
||||
for delta in (-1, 0, 1):
|
||||
with self.subTest(field=field, length=limit + delta):
|
||||
safe_field = field.replace("_", "-")
|
||||
flow = self.valid_flow(f"bounded-{safe_field}-{delta + 1}", "api", "FR-46")
|
||||
value = make_value(limit + delta)
|
||||
if field == "requirements":
|
||||
flow[field] = [value]
|
||||
required_frs = [value]
|
||||
else:
|
||||
flow[field] = value
|
||||
required_frs = ["FR-46"]
|
||||
self.write_inventory([flow])
|
||||
result = self.run_validator(required_frs)
|
||||
if delta <= 0:
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
else:
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("error[LIMIT_EXCEEDED]", result.stderr)
|
||||
|
||||
def test_collection_limits_cover_below_at_and_above_boundaries(self) -> None:
|
||||
module = load_validator_module()
|
||||
automated = self.write_evidence("evidence/shared/automated.txt")
|
||||
manual = self.write_evidence("evidence/shared/manual.md")
|
||||
|
||||
def direct_flow(index: int) -> dict[str, object]:
|
||||
return {
|
||||
"id": f"flow-{index:05d}",
|
||||
"type": "api",
|
||||
"requirements": ["FR-46"],
|
||||
"user_outcome": "A bounded flow succeeds.",
|
||||
"owner": "community-release",
|
||||
"status": "implemented",
|
||||
"capabilities": ["tools"],
|
||||
"evidence": {"automated": [automated], "manual": [manual]},
|
||||
}
|
||||
|
||||
all_flows = [direct_flow(index) for index in range(10_001)]
|
||||
for count in (9_999, 10_000, 10_001):
|
||||
with self.subTest(collection="flows", count=count):
|
||||
inventory = {
|
||||
"schema_version": 1,
|
||||
"product": "crank-community",
|
||||
"flows": all_flows[:count],
|
||||
}
|
||||
diagnostics, _ = module.validate_inventory(self.root, inventory, ["FR-46"])
|
||||
has_limit_error = any(
|
||||
item.code == "LIMIT_EXCEEDED" and item.pointer == "/flows"
|
||||
for item in diagnostics.ordered()
|
||||
)
|
||||
self.assertEqual(has_limit_error, count > 10_000)
|
||||
|
||||
for count in (63, 64, 65):
|
||||
with self.subTest(collection="requirements", count=count):
|
||||
flow = self.valid_flow(f"requirements-{count}", "api", "FR-1")
|
||||
flow["requirements"] = [f"FR-{index}" for index in range(1, count + 1)]
|
||||
self.write_inventory([flow])
|
||||
result = self.run_validator(["FR-1"])
|
||||
self.assertEqual(result.returncode == 0, count <= 64, result.stderr)
|
||||
|
||||
evidence_paths = [
|
||||
self.write_evidence(f"evidence/bounds/path-{index:02d}.txt")
|
||||
for index in range(65)
|
||||
]
|
||||
for count in (63, 64, 65):
|
||||
with self.subTest(collection="evidence", count=count):
|
||||
flow = self.valid_flow(f"evidence-{count}", "api", "FR-46")
|
||||
flow["evidence"] = {
|
||||
"automated": evidence_paths[:count],
|
||||
"manual": [manual],
|
||||
}
|
||||
self.write_inventory([flow])
|
||||
result = self.run_validator(["FR-46"])
|
||||
self.assertEqual(result.returncode == 0, count <= 64, result.stderr)
|
||||
|
||||
for count in (15, 16, 17):
|
||||
with self.subTest(collection="capability-bound", count=count):
|
||||
diagnostics = module.validate_string_list(
|
||||
[f"capability-{index}" for index in range(count)],
|
||||
"/capabilities",
|
||||
16,
|
||||
128,
|
||||
)
|
||||
has_limit_error = any(item.code == "LIMIT_EXCEEDED" for item in diagnostics)
|
||||
self.assertEqual(has_limit_error, count > 16)
|
||||
|
||||
def test_evidence_path_limit_covers_below_at_and_above_boundary(self) -> None:
|
||||
def path_of_length(length: int) -> str:
|
||||
segment_count = (length + 199) // 201
|
||||
character_count = length - (segment_count - 1)
|
||||
lengths: list[int] = []
|
||||
remaining = character_count
|
||||
for index in range(segment_count):
|
||||
slots_left = segment_count - index - 1
|
||||
current = min(200, remaining - slots_left)
|
||||
lengths.append(current)
|
||||
remaining -= current
|
||||
path = "/".join("p" * size for size in lengths)
|
||||
self.assertEqual(len(path), length)
|
||||
return path
|
||||
|
||||
manual = self.write_evidence("evidence/manual/path-limit.md")
|
||||
for length in (1_023, 1_024, 1_025):
|
||||
with self.subTest(length=length):
|
||||
evidence_path = path_of_length(length)
|
||||
if length <= 1_024:
|
||||
self.write_evidence(evidence_path)
|
||||
flow = self.valid_flow(f"path-limit-{length}", "api", "FR-46")
|
||||
flow["evidence"] = {
|
||||
"automated": [evidence_path],
|
||||
"manual": [manual],
|
||||
}
|
||||
self.write_inventory([flow])
|
||||
result = self.run_validator(["FR-46"])
|
||||
self.assertEqual(result.returncode == 0, length <= 1_024, result.stderr)
|
||||
|
||||
def test_semantic_validator_enforces_schema_shape_uniqueness_and_fr_format(self) -> None:
|
||||
cases: list[tuple[str, object, str]] = []
|
||||
|
||||
unknown_top = {
|
||||
"schema_version": 1,
|
||||
"product": "crank-community",
|
||||
"flows": [self.valid_flow("unknown-top", "api", "FR-46")],
|
||||
"unexpected": True,
|
||||
}
|
||||
cases.append(("unknown top-level field", unknown_top, "UNKNOWN_FIELD"))
|
||||
|
||||
unknown_flow = self.valid_flow("unknown-flow", "api", "FR-46")
|
||||
unknown_flow["unexpected"] = True
|
||||
cases.append(
|
||||
(
|
||||
"unknown flow field",
|
||||
{"schema_version": 1, "product": "crank-community", "flows": [unknown_flow]},
|
||||
"UNKNOWN_FIELD",
|
||||
)
|
||||
)
|
||||
|
||||
duplicate_requirement = self.valid_flow("duplicate-fr", "api", "FR-46")
|
||||
duplicate_requirement["requirements"] = ["FR-46", "FR-46"]
|
||||
cases.append(
|
||||
(
|
||||
"duplicate requirement",
|
||||
{
|
||||
"schema_version": 1,
|
||||
"product": "crank-community",
|
||||
"flows": [duplicate_requirement],
|
||||
},
|
||||
"DUPLICATE_LIST_ITEM",
|
||||
)
|
||||
)
|
||||
|
||||
malformed_requirement = self.valid_flow("malformed-fr", "api", "FR-46")
|
||||
malformed_requirement["requirements"] = ["FR-0"]
|
||||
cases.append(
|
||||
(
|
||||
"malformed requirement",
|
||||
{
|
||||
"schema_version": 1,
|
||||
"product": "crank-community",
|
||||
"flows": [malformed_requirement],
|
||||
},
|
||||
"INVALID_FORMAT",
|
||||
)
|
||||
)
|
||||
|
||||
unknown_evidence = self.valid_flow("unknown-evidence", "api", "FR-46")
|
||||
evidence = unknown_evidence["evidence"]
|
||||
self.assertIsInstance(evidence, dict)
|
||||
evidence["unexpected"] = []
|
||||
cases.append(
|
||||
(
|
||||
"unknown evidence field",
|
||||
{
|
||||
"schema_version": 1,
|
||||
"product": "crank-community",
|
||||
"flows": [unknown_evidence],
|
||||
},
|
||||
"UNKNOWN_FIELD",
|
||||
)
|
||||
)
|
||||
|
||||
for label, payload, expected_code in cases:
|
||||
with self.subTest(label=label):
|
||||
self.inventory_path.write_text(
|
||||
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = self.run_validator(["FR-46"])
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn(f"error[{expected_code}]", result.stderr)
|
||||
|
||||
def test_accepts_planned_modern_mcp_and_load_run_capabilities(self) -> None:
|
||||
capabilities = ["resources", "prompts", "tasks", "load_runs"]
|
||||
flows = [
|
||||
self.valid_flow(
|
||||
f"planned-{capability.replace('_', '-')}",
|
||||
"mcp" if capability != "load_runs" else "ui",
|
||||
f"FR-{index}",
|
||||
"planned",
|
||||
[capability],
|
||||
)
|
||||
for index, capability in enumerate(capabilities, start=47)
|
||||
]
|
||||
self.write_inventory(flows)
|
||||
|
||||
result = self.run_validator([f"FR-{index}" for index in range(47, 51)])
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
for capability in capabilities:
|
||||
self.assertNotIn(f"error[FORBIDDEN_COMMUNITY_CAPABILITY]: {capability}", result.stderr)
|
||||
|
||||
def test_rejects_forbidden_community_capabilities_despite_allow_text(self) -> None:
|
||||
forbidden = [
|
||||
"multi_workspace", # community-scope: allow=multi-workspace
|
||||
"enterprise_rbac", # community-scope: allow=enterprise
|
||||
"sso", # community-scope: allow=sso
|
||||
"non_rest_upstream", # community-scope: allow=non-rest-upstream
|
||||
"arbitrary_distributed_load_targets", # community-scope: allow=distributed-load-targets
|
||||
]
|
||||
|
||||
for index, capability in enumerate(forbidden, start=1):
|
||||
with self.subTest(capability=capability):
|
||||
flow = self.valid_flow(
|
||||
f"forbidden-{index}",
|
||||
"api",
|
||||
"FR-46",
|
||||
"planned",
|
||||
[capability],
|
||||
)
|
||||
flow["notes"] = f"community-scope: allow={capability}"
|
||||
self.write_inventory([flow])
|
||||
result = self.run_validator(["FR-46"])
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("error[FORBIDDEN_COMMUNITY_CAPABILITY]", result.stderr)
|
||||
self.assertIn(capability, result.stderr)
|
||||
|
||||
def test_scope_scanner_receives_inventory_files_explicitly_and_cannot_be_bypassed(self) -> None:
|
||||
clean_inventory = self.valid_flow(
|
||||
"planned-resources",
|
||||
"mcp",
|
||||
"FR-47",
|
||||
"planned",
|
||||
["resources"],
|
||||
)
|
||||
self.write_inventory([clean_inventory])
|
||||
explicit_files = [
|
||||
str(self.inventory_path.relative_to(self.root)),
|
||||
"docs/new-untracked-scope-note.md",
|
||||
]
|
||||
note = self.root / explicit_files[1]
|
||||
note.write_text("Community Resources remain planned.\n", encoding="utf-8")
|
||||
|
||||
clean = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(COMMUNITY_CHECKER),
|
||||
"--root",
|
||||
str(self.root),
|
||||
"--files",
|
||||
*explicit_files,
|
||||
],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
self.assertEqual(clean.returncode, 0, clean.stderr)
|
||||
self.assertIn("2 files scanned", clean.stdout)
|
||||
|
||||
note.write_text(
|
||||
"community-scope: allow=enterprise\n"
|
||||
"enterprise_rbac remains outside Community scope.\n", # community-scope: allow=enterprise
|
||||
encoding="utf-8",
|
||||
)
|
||||
rejected = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(COMMUNITY_CHECKER),
|
||||
"--root",
|
||||
str(self.root),
|
||||
"--files",
|
||||
*explicit_files,
|
||||
],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
self.assertNotEqual(rejected.returncode, 0)
|
||||
self.assertIn("docs/new-untracked-scope-note.md:2", rejected.stderr)
|
||||
self.assertIn("forbidden marker", rejected.stderr)
|
||||
self.assertIn("enterprise", rejected.stderr) # community-scope: allow=enterprise
|
||||
|
||||
def test_canonical_inventory_seed_does_not_promote_planned_work(self) -> None:
|
||||
for path in [
|
||||
CANONICAL_INVENTORY,
|
||||
CANONICAL_SCHEMA,
|
||||
]:
|
||||
self.assertTrue(path.is_file(), f"Missing canonical artifact: {path}")
|
||||
|
||||
inventory = json.loads(CANONICAL_INVENTORY.read_text(encoding="utf-8"))
|
||||
schema = json.loads(CANONICAL_SCHEMA.read_text(encoding="utf-8"))
|
||||
flow_schema = schema["$defs"]["flow"]
|
||||
evidence_paths = schema["$defs"]["evidencePaths"]
|
||||
self.assertEqual(
|
||||
schema["$schema"],
|
||||
"https://json-schema.org/draft/2020-12/schema",
|
||||
)
|
||||
self.assertEqual(schema["properties"]["flows"]["maxItems"], 10_000)
|
||||
self.assertEqual(
|
||||
set(flow_schema["properties"]["type"]["enum"]),
|
||||
{"ui", "api", "mcp"},
|
||||
)
|
||||
self.assertEqual(
|
||||
set(flow_schema["properties"]["status"]["enum"]),
|
||||
{"implemented", "planned", "gap", "blocked"},
|
||||
)
|
||||
self.assertEqual(evidence_paths["minItems"], 1)
|
||||
self.assertEqual(evidence_paths["maxItems"], 64)
|
||||
target_capabilities = {"resources", "prompts", "tasks", "load_runs"}
|
||||
matching_flows = [
|
||||
flow
|
||||
for flow in inventory["flows"]
|
||||
if target_capabilities.intersection(flow.get("capabilities", []))
|
||||
]
|
||||
observed_capabilities = {
|
||||
capability
|
||||
for flow in matching_flows
|
||||
for capability in flow.get("capabilities", [])
|
||||
if capability in target_capabilities
|
||||
}
|
||||
|
||||
self.assertEqual(observed_capabilities, target_capabilities)
|
||||
self.assertTrue(matching_flows)
|
||||
self.assertTrue(
|
||||
all(flow["status"] != "implemented" for flow in matching_flows),
|
||||
"Story 1.1 must not claim target capabilities are implemented.",
|
||||
)
|
||||
|
||||
def test_canonical_inventory_covers_all_requirements_and_required_surfaces(self) -> None:
|
||||
inventory = json.loads(CANONICAL_INVENTORY.read_text(encoding="utf-8"))
|
||||
required_surfaces = json.loads(REQUIRED_SURFACES.read_text(encoding="utf-8"))
|
||||
flows = {flow["id"]: flow for flow in inventory["flows"]}
|
||||
observed_requirements = {
|
||||
requirement for flow in flows.values() for requirement in flow["requirements"]
|
||||
}
|
||||
|
||||
self.assertEqual(
|
||||
{f"FR-{number}" for number in range(1, 55)} - observed_requirements,
|
||||
set(),
|
||||
)
|
||||
self.assertEqual(
|
||||
set(required_surfaces["required_flow_ids"]) - set(flows),
|
||||
set(),
|
||||
)
|
||||
for flow_id in required_surfaces["required_flow_ids"]:
|
||||
flow = flows[flow_id]
|
||||
self.assertIn(flow["status"], {"implemented", "blocked"})
|
||||
for evidence_kind in ("automated", "manual"):
|
||||
for evidence_path in flow["evidence"][evidence_kind]:
|
||||
resolved = ROOT / evidence_path
|
||||
self.assertTrue(resolved.is_file(), evidence_path)
|
||||
self.assertFalse(resolved.is_symlink(), evidence_path)
|
||||
|
||||
def test_tracked_documentation_describes_planned_scope(self) -> None:
|
||||
for documentation in [
|
||||
COMMUNITY_README,
|
||||
DOCS_INDEX,
|
||||
COMMUNITY_INTRO,
|
||||
COMMUNITY_README_EN,
|
||||
]:
|
||||
text = documentation.read_text(encoding="utf-8")
|
||||
self.assertIn("docs/capability-inventory.json", text)
|
||||
self.assertRegex(text.lower(), r"\b(planned|gap|blocked)\b")
|
||||
|
||||
def test_no_story_acceptance_scenario_remains_skipped(self) -> None:
|
||||
for name, method in vars(type(self)).items():
|
||||
if not name.startswith("test_") or name == self._testMethodName:
|
||||
continue
|
||||
self.assertFalse(
|
||||
getattr(method, "__unittest_skip__", False),
|
||||
f"Story acceptance scenario remains skipped: {name}",
|
||||
)
|
||||
|
||||
def test_canonical_validator_is_a_ci_and_release_prerequisite(self) -> None:
|
||||
justfile = JUSTFILE.read_text(encoding="utf-8")
|
||||
ci = CI_WORKFLOW.read_text(encoding="utf-8")
|
||||
release = RELEASE_WORKFLOW.read_text(encoding="utf-8")
|
||||
command = "scripts/validate-capability-inventory.py"
|
||||
|
||||
self.assertIn("capability-inventory-check:", justfile)
|
||||
self.assertIn("just capability-inventory-check", justfile)
|
||||
self.assertLess(ci.index("Run tooling unit tests"), ci.index(command))
|
||||
self.assertLess(ci.index(command), ci.index("Check Community scope"))
|
||||
self.assertIn("python3 --version", release)
|
||||
self.assertLess(release.index("Run tooling unit tests"), release.index(command))
|
||||
self.assertLess(release.index(command), release.index("Check Community scope"))
|
||||
self.assertLess(release.index("Check Community scope"), release.index("Build release binaries"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user