feat(openapi): complete upload preview and UI evidence

scripts/collect-capability-baseline.py

@@ -61,24 +61,25 @@ def load_report(path: Path) -> tuple[dict[str, Any], str]:
     return value, digest
 
 
-def iter_tests(value: Any):
+def iter_tests(value: Any, title: str | None = None):
     if isinstance(value, dict):
+        title = value.get("title") if isinstance(value.get("title"), str) else title
         tests = value.get("tests")
         if isinstance(tests, list):
             for test in tests:
                 if isinstance(test, dict):
-                    yield test
+                    yield test, title
         for key in ("suites", "specs"):
             children = value.get(key)
             if isinstance(children, list):
                 for child in children:
-                    yield from iter_tests(child)
+                    yield from iter_tests(child, title)
     elif isinstance(value, list):
         for item in value:
-            yield from iter_tests(item)
+            yield from iter_tests(item, title)
 
 
-def playwright_verdict(report: dict[str, Any]) -> tuple[str, dict[str, int]]:
+def playwright_verdict(report: dict[str, Any], required_titles: list[str]) -> tuple[str, dict[str, int]]:
     counts = {"passed": 0, "failed": 0, "flaky": 0, "skipped": 0, "not_run": 0}
     report_errors = report.get("errors", [])
     if not isinstance(report_errors, list):
@@ -91,7 +92,8 @@ def playwright_verdict(report: dict[str, Any]) -> tuple[str, dict[str, int]]:
     if not tests:
         counts["not_run"] = 1
         return "not_run", counts
-    for test in tests:
+    required = {title: False for title in required_titles}
+    for test, title in tests:
         status = test.get("status")
         results = test.get("results") if isinstance(test.get("results"), list) else []
         result_statuses = [result.get("status") for result in results if isinstance(result, dict)]
@@ -109,8 +111,12 @@ def playwright_verdict(report: dict[str, Any]) -> tuple[str, dict[str, int]]:
             counts["failed"] += 1
         elif results and all(result_status == "passed" for result_status in result_statuses):
             counts["passed"] += 1
+            if title in required:
+                required[title] = True
         else:
             counts["not_run"] += 1
+    if not all(required.values()):
+        counts["failed"] += len([title for title, passed in required.items() if not passed])
     if counts["failed"]:
         return "fail", counts
     if counts["flaky"]:
@@ -134,7 +140,7 @@ def validate_labels(args: argparse.Namespace) -> None:
 def collect_playwright(args: argparse.Namespace) -> dict[str, Any]:
     validate_labels(args)
     report, digest = load_report(Path(args.report))
-    verdict, counts = playwright_verdict(report)
+    verdict, counts = playwright_verdict(report, args.required_test)
     return {
         "accepted": verdict == "pass",
         "collector": "capability-baseline-collector-v1",
@@ -193,6 +199,7 @@ def parser() -> argparse.ArgumentParser:
     playwright.add_argument("--source-revision", required=True)
     playwright.add_argument("--environment-class", required=True)
     playwright.add_argument("--flow-id", action="append", required=True)
+    playwright.add_argument("--required-test", action="append", default=[])
     command = subparsers.add_parser("command-report")
     command.add_argument("--report", required=True)
     command.add_argument("--output", required=True)