fix(ci): scope OpenAPI evidence to required tests

scripts/collect-capability-baseline.py

@@ -90,26 +90,52 @@ def playwright_verdict(report: dict[str, Any], required_titles: list[str]) -> tu
         return "fail", counts
     tests = list(iter_tests(report))
     if not tests:
+        if required_titles:
+            counts["failed"] = len(set(required_titles))
+            return "fail", counts
         counts["not_run"] = 1
         return "not_run", counts
     required = {title: False for title in required_titles}
     for test, title in tests:
+        if required and title not in required:
+            continue
         status = test.get("status")
-        results = test.get("results") if isinstance(test.get("results"), list) else []
-        result_statuses = [result.get("status") for result in results if isinstance(result, dict)]
-        retries = [result.get("retry", 0) for result in results if isinstance(result, dict)]
-        if len(result_statuses) != len(results):
+        results = test.get("results")
+        if not isinstance(results, list):
             counts["not_run"] += 1
             continue
-        if status == "flaky" or any(isinstance(retry, int) and retry > 0 for retry in retries):
-            counts["flaky"] += 1
+        result_statuses: list[Any] = []
+        retries: list[int] = []
+        malformed = False
+        for result in results:
+            if not isinstance(result, dict):
+                malformed = True
+                break
+            result_status = result.get("status")
+            retry = result.get("retry", 0)
+            if result_status not in ("passed", "failed", "timedOut", "skipped", "interrupted") \
+                or type(retry) is not int or retry < 0:
+                malformed = True
+                break
+            result_statuses.append(result_status)
+            retries.append(retry)
+        if malformed:
+            counts["not_run"] += 1
+            continue
+        final_status = result_statuses[-1] if result_statuses else None
+        if status in ("unexpected", "failed", "timedOut", "interrupted") \
+            or final_status in ("failed", "timedOut", "interrupted"):
+            counts["failed"] += 1
+        elif status == "flaky":
+            if final_status == "passed" and any(retry > 0 for retry in retries):
+                counts["flaky"] += 1
+            else:
+                counts["not_run"] += 1
         elif status == "skipped" or (not results and status in ("skipped", "expected")):
             counts["skipped"] += 1
-        elif status in ("unexpected", "failed", "timedOut", "interrupted") or any(
-            result_status in ("failed", "timedOut", "interrupted") for result_status in result_statuses
-        ):
-            counts["failed"] += 1
-        elif results and all(result_status == "passed" for result_status in result_statuses):
+        elif any(retry > 0 for retry in retries):
+            counts["flaky"] += 1
+        elif status == "expected" and results and all(result_status == "passed" for result_status in result_statuses):
             counts["passed"] += 1
             if title in required:
                 required[title] = True

tests/unit/test_collect_capability_baseline.py

@@ -57,6 +57,7 @@ class CapabilityBaselineCollectorTests(unittest.TestCase):
     def test_required_openapi_tests_fail_closed_when_missing_skipped_or_flaky(self) -> None:
         required = ["OpenAPI required scenario"]
         reports = [
+            {"suites": []},
             {"suites": [{"specs": [{"title": "another scenario", "tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}]}]}]},
             {"suites": [{"specs": [{"title": required[0], "tests": [{"status": "skipped", "results": []}]}]}]},
             {"suites": [{"specs": [{"title": required[0], "tests": [{"status": "flaky", "results": [{"status": "failed", "retry": 0}, {"status": "passed", "retry": 1}]}]}]}]},
@@ -70,6 +71,73 @@ class CapabilityBaselineCollectorTests(unittest.TestCase):
                 self.assertEqual(candidate["execution_verdict"], "fail")
                 self.assertFalse(candidate["accepted"])
 
+    def test_required_scope_ignores_unrelated_flaky_tests(self) -> None:
+        required = "OpenAPI required scenario"
+        report = {
+            "suites": [
+                {
+                    "specs": [
+                        {
+                            "title": required,
+                            "tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}],
+                        },
+                        {
+                            "title": "Unrelated wizard scenario",
+                            "tests": [
+                                {
+                                    "status": "flaky",
+                                    "results": [
+                                        {"status": "failed", "retry": 0},
+                                        {"status": "passed", "retry": 1},
+                                    ],
+                                }
+                            ],
+                        },
+                        {"title": "Unrelated failure", "tests": [{"status": "unexpected", "results": [{"status": "failed", "retry": 0}]}]},
+                        {"title": "Unrelated skip", "tests": [{"status": "skipped", "results": []}]},
+                        {"title": "Unrelated malformed", "tests": [{"status": "expected", "results": None}]},
+                    ]
+                }
+            ]
+        }
+        result, output, temporary = self.run_playwright(report, [required])
+        self.addCleanup(temporary.cleanup)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        candidate = json.loads(output.read_text(encoding="utf-8"))
+        self.assertEqual(candidate["execution_verdict"], "pass")
+        self.assertTrue(candidate["accepted"])
+        self.assertEqual(candidate["summary"], {"passed": 1, "failed": 0, "flaky": 0, "skipped": 0, "not_run": 0})
+
+    def test_final_failure_after_retry_is_failed_not_flaky(self) -> None:
+        for status in ("unexpected", "flaky"):
+            with self.subTest(status=status):
+                report = {
+                    "suites": [
+                        {
+                            "specs": [
+                                {
+                                    "tests": [
+                                        {
+                                            "status": status,
+                                            "results": [
+                                                {"status": "failed", "retry": 0},
+                                                {"status": "failed", "retry": 1},
+                                            ],
+                                        }
+                                    ]
+                                }
+                            ]
+                        }
+                    ]
+                }
+                result, output, temporary = self.run_playwright(report)
+                self.addCleanup(temporary.cleanup)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                candidate = json.loads(output.read_text(encoding="utf-8"))
+                self.assertEqual(candidate["execution_verdict"], "fail")
+                self.assertEqual(candidate["summary"]["failed"], 1)
+                self.assertEqual(candidate["summary"]["flaky"], 0)
+
     def test_raw_report_content_never_reaches_candidate_or_error(self) -> None:
         canary = "Bearer secret-canary /home/private/workspace https://private.invalid?q=secret"
         report = {"suites": [], "errors": [{"message": canary}], "stdout": [canary]}
@@ -84,6 +152,8 @@ class CapabilityBaselineCollectorTests(unittest.TestCase):
         reports = [
             {"errors": [{"message": "fatal"}], "suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed"}]}]}]}]},
             {"suites": [{"specs": [{"tests": [{"status": "expected", "results": ["not-an-object"]}]}]}]},
+            {"suites": [{"specs": [{"tests": [{"status": "expected", "results": None}]}]}]},
+            {"suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed", "retry": True}]}]}]}]},
         ]
         for report in reports:
             with self.subTest(report=report):