100 lines
4.0 KiB
Python
100 lines
4.0 KiB
Python
import json
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCHEMA = ROOT / "docs" / "schemas" / "capability-baseline.schema.json"
|
|
VALIDATOR = ROOT / "scripts" / "validate-capability-run.py"
|
|
|
|
|
|
def candidate() -> dict[str, object]:
|
|
return {
|
|
"id": "run-ui-playwright-0123456789ab",
|
|
"command_id": "ui-playwright",
|
|
"flow_ids": ["openapi-upload-ui"],
|
|
"execution_verdict": "pass",
|
|
"evidence_mode": "automated",
|
|
"accepted": True,
|
|
"source_report_sha256": "a" * 64,
|
|
"source_revision": "0" * 40,
|
|
"environment_class": "ci",
|
|
"collector": "capability-baseline-collector-v1",
|
|
"summary": {"passed": 1},
|
|
}
|
|
|
|
|
|
class CapabilityRunValidatorTests(unittest.TestCase):
|
|
def validate(self, contents: str | bytes, require_accepted: bool = False) -> subprocess.CompletedProcess[str]:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
candidate_path = Path(directory) / "candidate.json"
|
|
if isinstance(contents, bytes):
|
|
candidate_path.write_bytes(contents)
|
|
else:
|
|
candidate_path.write_text(contents, encoding="utf-8")
|
|
command = ["python3", str(VALIDATOR), "--schema", str(SCHEMA), "--candidate", str(candidate_path)]
|
|
if require_accepted:
|
|
command.append("--require-accepted")
|
|
return subprocess.run(
|
|
command,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
|
|
def test_valid_candidate_matches_run_definition(self) -> None:
|
|
result = self.validate(json.dumps(candidate()))
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertIn("matches $defs/run", result.stdout)
|
|
|
|
def test_missing_required_property_is_rejected(self) -> None:
|
|
value = candidate()
|
|
value.pop("id")
|
|
result = self.validate(json.dumps(value))
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("missing required property id", result.stderr)
|
|
|
|
def test_extra_property_is_rejected(self) -> None:
|
|
value = candidate()
|
|
value["raw_report"] = "must-not-be-accepted"
|
|
result = self.validate(json.dumps(value))
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("unexpected property raw_report", result.stderr)
|
|
|
|
def test_wrong_type_and_pattern_are_rejected(self) -> None:
|
|
type_error = candidate()
|
|
type_error["accepted"] = "true"
|
|
pattern_error = candidate()
|
|
pattern_error["source_revision"] = "not-a-revision"
|
|
for value, expected in ((type_error, "expected boolean"), (pattern_error, "string does not match pattern")):
|
|
with self.subTest(expected=expected):
|
|
result = self.validate(json.dumps(value))
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn(expected, result.stderr)
|
|
|
|
def test_duplicate_json_key_is_rejected(self) -> None:
|
|
encoded = json.dumps(candidate())
|
|
duplicate = encoded.replace('"id": "run-ui-playwright-0123456789ab",', '"id": "run-ui-playwright-0123456789ab", "id": "other",')
|
|
result = self.validate(duplicate)
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("duplicate JSON key", result.stderr)
|
|
|
|
def test_oversized_candidate_is_rejected(self) -> None:
|
|
result = self.validate(b" " * (65_536 + 1))
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("input exceeds 64 KiB", result.stderr)
|
|
|
|
def test_require_accepted_rejects_schema_valid_negative_evidence(self) -> None:
|
|
value = candidate()
|
|
value["accepted"] = False
|
|
value["execution_verdict"] = "fail"
|
|
result = self.validate(json.dumps(value), require_accepted=True)
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("accepted evidence is required", result.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|