fix(ci): scope OpenAPI evidence to required tests

2026-09-01 12:59:04 +03:00
parent 6c5d0c62d0
commit 66b9bfd875
2 changed files with 107 additions and 11 deletions
@@ -57,6 +57,7 @@ class CapabilityBaselineCollectorTests(unittest.TestCase):
    def test_required_openapi_tests_fail_closed_when_missing_skipped_or_flaky(self) -> None:
        required = ["OpenAPI required scenario"]
        reports = [
+            {"suites": []},
            {"suites": [{"specs": [{"title": "another scenario", "tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}]}]}]},
            {"suites": [{"specs": [{"title": required[0], "tests": [{"status": "skipped", "results": []}]}]}]},
            {"suites": [{"specs": [{"title": required[0], "tests": [{"status": "flaky", "results": [{"status": "failed", "retry": 0}, {"status": "passed", "retry": 1}]}]}]}]},
@@ -70,6 +71,73 @@ class CapabilityBaselineCollectorTests(unittest.TestCase):
                self.assertEqual(candidate["execution_verdict"], "fail")
                self.assertFalse(candidate["accepted"])

+    def test_required_scope_ignores_unrelated_flaky_tests(self) -> None:
+        required = "OpenAPI required scenario"
+        report = {
+            "suites": [
+                {
+                    "specs": [
+                        {
+                            "title": required,
+                            "tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}],
+                        },
+                        {
+                            "title": "Unrelated wizard scenario",
+                            "tests": [
+                                {
+                                    "status": "flaky",
+                                    "results": [
+                                        {"status": "failed", "retry": 0},
+                                        {"status": "passed", "retry": 1},
+                                    ],
+                                }
+                            ],
+                        },
+                        {"title": "Unrelated failure", "tests": [{"status": "unexpected", "results": [{"status": "failed", "retry": 0}]}]},
+                        {"title": "Unrelated skip", "tests": [{"status": "skipped", "results": []}]},
+                        {"title": "Unrelated malformed", "tests": [{"status": "expected", "results": None}]},
+                    ]
+                }
+            ]
+        }
+        result, output, temporary = self.run_playwright(report, [required])
+        self.addCleanup(temporary.cleanup)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        candidate = json.loads(output.read_text(encoding="utf-8"))
+        self.assertEqual(candidate["execution_verdict"], "pass")
+        self.assertTrue(candidate["accepted"])
+        self.assertEqual(candidate["summary"], {"passed": 1, "failed": 0, "flaky": 0, "skipped": 0, "not_run": 0})
+
+    def test_final_failure_after_retry_is_failed_not_flaky(self) -> None:
+        for status in ("unexpected", "flaky"):
+            with self.subTest(status=status):
+                report = {
+                    "suites": [
+                        {
+                            "specs": [
+                                {
+                                    "tests": [
+                                        {
+                                            "status": status,
+                                            "results": [
+                                                {"status": "failed", "retry": 0},
+                                                {"status": "failed", "retry": 1},
+                                            ],
+                                        }
+                                    ]
+                                }
+                            ]
+                        }
+                    ]
+                }
+                result, output, temporary = self.run_playwright(report)
+                self.addCleanup(temporary.cleanup)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                candidate = json.loads(output.read_text(encoding="utf-8"))
+                self.assertEqual(candidate["execution_verdict"], "fail")
+                self.assertEqual(candidate["summary"]["failed"], 1)
+                self.assertEqual(candidate["summary"]["flaky"], 0)
+
    def test_raw_report_content_never_reaches_candidate_or_error(self) -> None:
        canary = "Bearer secret-canary /home/private/workspace https://private.invalid?q=secret"
        report = {"suites": [], "errors": [{"message": canary}], "stdout": [canary]}
@@ -84,6 +152,8 @@ class CapabilityBaselineCollectorTests(unittest.TestCase):
        reports = [
            {"errors": [{"message": "fatal"}], "suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed"}]}]}]}]},
            {"suites": [{"specs": [{"tests": [{"status": "expected", "results": ["not-an-object"]}]}]}]},
+            {"suites": [{"specs": [{"tests": [{"status": "expected", "results": None}]}]}]},
+            {"suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed", "retry": True}]}]}]}]},
        ]
        for report in reports:
            with self.subTest(report=report):