From 66b9bfd87502321628f096738f42e39227f6e740 Mon Sep 17 00:00:00 2001 From: bsodfather Date: Tue, 1 Sep 2026 12:59:04 +0300 Subject: [PATCH] fix(ci): scope OpenAPI evidence to required tests --- scripts/collect-capability-baseline.py | 48 ++++++++++--- .../unit/test_collect_capability_baseline.py | 70 +++++++++++++++++++ 2 files changed, 107 insertions(+), 11 deletions(-) diff --git a/scripts/collect-capability-baseline.py b/scripts/collect-capability-baseline.py index c432551..ddac1cb 100644 --- a/scripts/collect-capability-baseline.py +++ b/scripts/collect-capability-baseline.py @@ -90,26 +90,52 @@ def playwright_verdict(report: dict[str, Any], required_titles: list[str]) -> tu return "fail", counts tests = list(iter_tests(report)) if not tests: + if required_titles: + counts["failed"] = len(set(required_titles)) + return "fail", counts counts["not_run"] = 1 return "not_run", counts required = {title: False for title in required_titles} for test, title in tests: + if required and title not in required: + continue status = test.get("status") - results = test.get("results") if isinstance(test.get("results"), list) else [] - result_statuses = [result.get("status") for result in results if isinstance(result, dict)] - retries = [result.get("retry", 0) for result in results if isinstance(result, dict)] - if len(result_statuses) != len(results): + results = test.get("results") + if not isinstance(results, list): counts["not_run"] += 1 continue - if status == "flaky" or any(isinstance(retry, int) and retry > 0 for retry in retries): - counts["flaky"] += 1 + result_statuses: list[Any] = [] + retries: list[int] = [] + malformed = False + for result in results: + if not isinstance(result, dict): + malformed = True + break + result_status = result.get("status") + retry = result.get("retry", 0) + if result_status not in ("passed", "failed", "timedOut", "skipped", "interrupted") \ + or type(retry) is not int or retry < 0: + malformed = True + break + result_statuses.append(result_status) + retries.append(retry) + if malformed: + counts["not_run"] += 1 + continue + final_status = result_statuses[-1] if result_statuses else None + if status in ("unexpected", "failed", "timedOut", "interrupted") \ + or final_status in ("failed", "timedOut", "interrupted"): + counts["failed"] += 1 + elif status == "flaky": + if final_status == "passed" and any(retry > 0 for retry in retries): + counts["flaky"] += 1 + else: + counts["not_run"] += 1 elif status == "skipped" or (not results and status in ("skipped", "expected")): counts["skipped"] += 1 - elif status in ("unexpected", "failed", "timedOut", "interrupted") or any( - result_status in ("failed", "timedOut", "interrupted") for result_status in result_statuses - ): - counts["failed"] += 1 - elif results and all(result_status == "passed" for result_status in result_statuses): + elif any(retry > 0 for retry in retries): + counts["flaky"] += 1 + elif status == "expected" and results and all(result_status == "passed" for result_status in result_statuses): counts["passed"] += 1 if title in required: required[title] = True diff --git a/tests/unit/test_collect_capability_baseline.py b/tests/unit/test_collect_capability_baseline.py index 18aa4dc..f006fc5 100644 --- a/tests/unit/test_collect_capability_baseline.py +++ b/tests/unit/test_collect_capability_baseline.py @@ -57,6 +57,7 @@ class CapabilityBaselineCollectorTests(unittest.TestCase): def test_required_openapi_tests_fail_closed_when_missing_skipped_or_flaky(self) -> None: required = ["OpenAPI required scenario"] reports = [ + {"suites": []}, {"suites": [{"specs": [{"title": "another scenario", "tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}]}]}]}, {"suites": [{"specs": [{"title": required[0], "tests": [{"status": "skipped", "results": []}]}]}]}, {"suites": [{"specs": [{"title": required[0], "tests": [{"status": "flaky", "results": [{"status": "failed", "retry": 0}, {"status": "passed", "retry": 1}]}]}]}]}, @@ -70,6 +71,73 @@ class CapabilityBaselineCollectorTests(unittest.TestCase): self.assertEqual(candidate["execution_verdict"], "fail") self.assertFalse(candidate["accepted"]) + def test_required_scope_ignores_unrelated_flaky_tests(self) -> None: + required = "OpenAPI required scenario" + report = { + "suites": [ + { + "specs": [ + { + "title": required, + "tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}], + }, + { + "title": "Unrelated wizard scenario", + "tests": [ + { + "status": "flaky", + "results": [ + {"status": "failed", "retry": 0}, + {"status": "passed", "retry": 1}, + ], + } + ], + }, + {"title": "Unrelated failure", "tests": [{"status": "unexpected", "results": [{"status": "failed", "retry": 0}]}]}, + {"title": "Unrelated skip", "tests": [{"status": "skipped", "results": []}]}, + {"title": "Unrelated malformed", "tests": [{"status": "expected", "results": None}]}, + ] + } + ] + } + result, output, temporary = self.run_playwright(report, [required]) + self.addCleanup(temporary.cleanup) + self.assertEqual(result.returncode, 0, result.stderr) + candidate = json.loads(output.read_text(encoding="utf-8")) + self.assertEqual(candidate["execution_verdict"], "pass") + self.assertTrue(candidate["accepted"]) + self.assertEqual(candidate["summary"], {"passed": 1, "failed": 0, "flaky": 0, "skipped": 0, "not_run": 0}) + + def test_final_failure_after_retry_is_failed_not_flaky(self) -> None: + for status in ("unexpected", "flaky"): + with self.subTest(status=status): + report = { + "suites": [ + { + "specs": [ + { + "tests": [ + { + "status": status, + "results": [ + {"status": "failed", "retry": 0}, + {"status": "failed", "retry": 1}, + ], + } + ] + } + ] + } + ] + } + result, output, temporary = self.run_playwright(report) + self.addCleanup(temporary.cleanup) + self.assertEqual(result.returncode, 0, result.stderr) + candidate = json.loads(output.read_text(encoding="utf-8")) + self.assertEqual(candidate["execution_verdict"], "fail") + self.assertEqual(candidate["summary"]["failed"], 1) + self.assertEqual(candidate["summary"]["flaky"], 0) + def test_raw_report_content_never_reaches_candidate_or_error(self) -> None: canary = "Bearer secret-canary /home/private/workspace https://private.invalid?q=secret" report = {"suites": [], "errors": [{"message": canary}], "stdout": [canary]} @@ -84,6 +152,8 @@ class CapabilityBaselineCollectorTests(unittest.TestCase): reports = [ {"errors": [{"message": "fatal"}], "suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed"}]}]}]}]}, {"suites": [{"specs": [{"tests": [{"status": "expected", "results": ["not-an-object"]}]}]}]}, + {"suites": [{"specs": [{"tests": [{"status": "expected", "results": None}]}]}]}, + {"suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed", "retry": True}]}]}]}]}, ] for report in reports: with self.subTest(report=report):