feat(openapi): complete upload preview and UI evidence

tests/unit/test_collect_capability_baseline.py

@@ -10,14 +10,17 @@ COLLECTOR = ROOT / "scripts" / "collect-capability-baseline.py"
 
 
 class CapabilityBaselineCollectorTests(unittest.TestCase):
-    def run_playwright(self, report: object) -> tuple[subprocess.CompletedProcess[str], Path, tempfile.TemporaryDirectory[str]]:
+    def run_playwright(self, report: object, required: list[str] | None = None) -> tuple[subprocess.CompletedProcess[str], Path, tempfile.TemporaryDirectory[str]]:
         temporary = tempfile.TemporaryDirectory()
         root = Path(temporary.name)
         report_path = root / "report.json"
         output_path = root / "candidate.json"
         report_path.write_text(json.dumps(report), encoding="utf-8")
+        command = ["python3", str(COLLECTOR), "playwright", "--report", str(report_path), "--output", str(output_path), "--source-revision", "0" * 40, "--environment-class", "community-test", "--flow-id", "ui-operations"]
+        for title in required or []:
+            command.extend(["--required-test", title])
         result = subprocess.run(
-            ["python3", str(COLLECTOR), "playwright", "--report", str(report_path), "--output", str(output_path), "--source-revision", "0" * 40, "--environment-class", "community-test", "--flow-id", "ui-operations"],
+            command,
             text=True,
             capture_output=True,
             check=False,
@@ -51,6 +54,22 @@ class CapabilityBaselineCollectorTests(unittest.TestCase):
         self.assertEqual(result.returncode, 0, result.stderr)
         self.assertEqual(json.loads(output.read_text(encoding="utf-8"))["execution_verdict"], "skipped")
 
+    def test_required_openapi_tests_fail_closed_when_missing_skipped_or_flaky(self) -> None:
+        required = ["OpenAPI required scenario"]
+        reports = [
+            {"suites": [{"specs": [{"title": "another scenario", "tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}]}]}]},
+            {"suites": [{"specs": [{"title": required[0], "tests": [{"status": "skipped", "results": []}]}]}]},
+            {"suites": [{"specs": [{"title": required[0], "tests": [{"status": "flaky", "results": [{"status": "failed", "retry": 0}, {"status": "passed", "retry": 1}]}]}]}]},
+        ]
+        for report in reports:
+            with self.subTest(report=report):
+                result, output, temporary = self.run_playwright(report, required)
+                self.addCleanup(temporary.cleanup)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                candidate = json.loads(output.read_text(encoding="utf-8"))
+                self.assertEqual(candidate["execution_verdict"], "fail")
+                self.assertFalse(candidate["accepted"])
+
     def test_raw_report_content_never_reaches_candidate_or_error(self) -> None:
         canary = "Bearer secret-canary /home/private/workspace https://private.invalid?q=secret"
         report = {"suites": [], "errors": [{"message": canary}], "stdout": [canary]}

tests/unit/test_validate_capability_run.py

@@ -0,0 +1,99 @@
+import json
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SCHEMA = ROOT / "docs" / "schemas" / "capability-baseline.schema.json"
+VALIDATOR = ROOT / "scripts" / "validate-capability-run.py"
+
+
+def candidate() -> dict[str, object]:
+    return {
+        "id": "run-ui-playwright-0123456789ab",
+        "command_id": "ui-playwright",
+        "flow_ids": ["openapi-upload-ui"],
+        "execution_verdict": "pass",
+        "evidence_mode": "automated",
+        "accepted": True,
+        "source_report_sha256": "a" * 64,
+        "source_revision": "0" * 40,
+        "environment_class": "ci",
+        "collector": "capability-baseline-collector-v1",
+        "summary": {"passed": 1},
+    }
+
+
+class CapabilityRunValidatorTests(unittest.TestCase):
+    def validate(self, contents: str | bytes, require_accepted: bool = False) -> subprocess.CompletedProcess[str]:
+        with tempfile.TemporaryDirectory() as directory:
+            candidate_path = Path(directory) / "candidate.json"
+            if isinstance(contents, bytes):
+                candidate_path.write_bytes(contents)
+            else:
+                candidate_path.write_text(contents, encoding="utf-8")
+            command = ["python3", str(VALIDATOR), "--schema", str(SCHEMA), "--candidate", str(candidate_path)]
+            if require_accepted:
+                command.append("--require-accepted")
+            return subprocess.run(
+                command,
+                text=True,
+                capture_output=True,
+                check=False,
+            )
+
+    def test_valid_candidate_matches_run_definition(self) -> None:
+        result = self.validate(json.dumps(candidate()))
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("matches $defs/run", result.stdout)
+
+    def test_missing_required_property_is_rejected(self) -> None:
+        value = candidate()
+        value.pop("id")
+        result = self.validate(json.dumps(value))
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("missing required property id", result.stderr)
+
+    def test_extra_property_is_rejected(self) -> None:
+        value = candidate()
+        value["raw_report"] = "must-not-be-accepted"
+        result = self.validate(json.dumps(value))
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("unexpected property raw_report", result.stderr)
+
+    def test_wrong_type_and_pattern_are_rejected(self) -> None:
+        type_error = candidate()
+        type_error["accepted"] = "true"
+        pattern_error = candidate()
+        pattern_error["source_revision"] = "not-a-revision"
+        for value, expected in ((type_error, "expected boolean"), (pattern_error, "string does not match pattern")):
+            with self.subTest(expected=expected):
+                result = self.validate(json.dumps(value))
+                self.assertNotEqual(result.returncode, 0)
+                self.assertIn(expected, result.stderr)
+
+    def test_duplicate_json_key_is_rejected(self) -> None:
+        encoded = json.dumps(candidate())
+        duplicate = encoded.replace('"id": "run-ui-playwright-0123456789ab",', '"id": "run-ui-playwright-0123456789ab", "id": "other",')
+        result = self.validate(duplicate)
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("duplicate JSON key", result.stderr)
+
+    def test_oversized_candidate_is_rejected(self) -> None:
+        result = self.validate(b" " * (65_536 + 1))
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("input exceeds 64 KiB", result.stderr)
+
+    def test_require_accepted_rejects_schema_valid_negative_evidence(self) -> None:
+        value = candidate()
+        value["accepted"] = False
+        value["execution_verdict"] = "fail"
+        result = self.validate(json.dumps(value), require_accepted=True)
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("accepted evidence is required", result.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main()