import json import subprocess import tempfile import unittest from pathlib import Path ROOT = Path(__file__).resolve().parents[2] SCHEMA = ROOT / "docs" / "schemas" / "capability-baseline.schema.json" VALIDATOR = ROOT / "scripts" / "validate-capability-run.py" def candidate() -> dict[str, object]: return { "id": "run-ui-playwright-0123456789ab", "command_id": "ui-playwright", "flow_ids": ["openapi-upload-ui"], "execution_verdict": "pass", "evidence_mode": "automated", "accepted": True, "source_report_sha256": "a" * 64, "source_revision": "0" * 40, "environment_class": "ci", "collector": "capability-baseline-collector-v1", "summary": {"passed": 1}, } class CapabilityRunValidatorTests(unittest.TestCase): def validate(self, contents: str | bytes, require_accepted: bool = False) -> subprocess.CompletedProcess[str]: with tempfile.TemporaryDirectory() as directory: candidate_path = Path(directory) / "candidate.json" if isinstance(contents, bytes): candidate_path.write_bytes(contents) else: candidate_path.write_text(contents, encoding="utf-8") command = ["python3", str(VALIDATOR), "--schema", str(SCHEMA), "--candidate", str(candidate_path)] if require_accepted: command.append("--require-accepted") return subprocess.run( command, text=True, capture_output=True, check=False, ) def test_valid_candidate_matches_run_definition(self) -> None: result = self.validate(json.dumps(candidate())) self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("matches $defs/run", result.stdout) def test_missing_required_property_is_rejected(self) -> None: value = candidate() value.pop("id") result = self.validate(json.dumps(value)) self.assertNotEqual(result.returncode, 0) self.assertIn("missing required property id", result.stderr) def test_extra_property_is_rejected(self) -> None: value = candidate() value["raw_report"] = "must-not-be-accepted" result = self.validate(json.dumps(value)) self.assertNotEqual(result.returncode, 0) self.assertIn("unexpected property raw_report", result.stderr) def test_wrong_type_and_pattern_are_rejected(self) -> None: type_error = candidate() type_error["accepted"] = "true" pattern_error = candidate() pattern_error["source_revision"] = "not-a-revision" for value, expected in ((type_error, "expected boolean"), (pattern_error, "string does not match pattern")): with self.subTest(expected=expected): result = self.validate(json.dumps(value)) self.assertNotEqual(result.returncode, 0) self.assertIn(expected, result.stderr) def test_duplicate_json_key_is_rejected(self) -> None: encoded = json.dumps(candidate()) duplicate = encoded.replace('"id": "run-ui-playwright-0123456789ab",', '"id": "run-ui-playwright-0123456789ab", "id": "other",') result = self.validate(duplicate) self.assertNotEqual(result.returncode, 0) self.assertIn("duplicate JSON key", result.stderr) def test_oversized_candidate_is_rejected(self) -> None: result = self.validate(b" " * (65_536 + 1)) self.assertNotEqual(result.returncode, 0) self.assertIn("input exceeds 64 KiB", result.stderr) def test_require_accepted_rejects_schema_valid_negative_evidence(self) -> None: value = candidate() value["accepted"] = False value["execution_verdict"] = "fail" result = self.validate(json.dumps(value), require_accepted=True) self.assertNotEqual(result.returncode, 0) self.assertIn("accepted evidence is required", result.stderr) if __name__ == "__main__": unittest.main()