211 lines
11 KiB
Python
211 lines
11 KiB
Python
import json
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
COLLECTOR = ROOT / "scripts" / "collect-capability-baseline.py"
|
|
|
|
|
|
class CapabilityBaselineCollectorTests(unittest.TestCase):
|
|
def run_playwright(self, report: object, required: list[str] | None = None) -> tuple[subprocess.CompletedProcess[str], Path, tempfile.TemporaryDirectory[str]]:
|
|
temporary = tempfile.TemporaryDirectory()
|
|
root = Path(temporary.name)
|
|
report_path = root / "report.json"
|
|
output_path = root / "candidate.json"
|
|
report_path.write_text(json.dumps(report), encoding="utf-8")
|
|
command = ["python3", str(COLLECTOR), "playwright", "--report", str(report_path), "--output", str(output_path), "--source-revision", "0" * 40, "--environment-class", "community-test", "--flow-id", "ui-operations"]
|
|
for title in required or []:
|
|
command.extend(["--required-test", title])
|
|
result = subprocess.run(
|
|
command,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
return result, output_path, temporary
|
|
|
|
def test_stable_playwright_pass_is_collected(self) -> None:
|
|
report = {"suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}]}]}]}
|
|
result, output, temporary = self.run_playwright(report)
|
|
self.addCleanup(temporary.cleanup)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
candidate = json.loads(output.read_text(encoding="utf-8"))
|
|
self.assertEqual(candidate["execution_verdict"], "pass")
|
|
self.assertTrue(candidate["accepted"])
|
|
self.assertEqual(candidate["evidence_mode"], "automated")
|
|
self.assertRegex(candidate["source_report_sha256"], "^[0-9a-f]{64}$")
|
|
|
|
def test_retry_pass_is_flaky_and_not_accepted(self) -> None:
|
|
report = {"suites": [{"specs": [{"tests": [{"status": "flaky", "results": [{"status": "failed", "retry": 0}, {"status": "passed", "retry": 1}]}]}]}]}
|
|
result, output, temporary = self.run_playwright(report)
|
|
self.addCleanup(temporary.cleanup)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
candidate = json.loads(output.read_text(encoding="utf-8"))
|
|
self.assertEqual(candidate["execution_verdict"], "flaky")
|
|
self.assertFalse(candidate["accepted"])
|
|
|
|
def test_skipped_and_missing_results_are_not_pass(self) -> None:
|
|
report = {"suites": [{"specs": [{"tests": [{"status": "skipped", "results": []}]}]}]}
|
|
result, output, temporary = self.run_playwright(report)
|
|
self.addCleanup(temporary.cleanup)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(json.loads(output.read_text(encoding="utf-8"))["execution_verdict"], "skipped")
|
|
|
|
def test_required_openapi_tests_fail_closed_when_missing_skipped_or_flaky(self) -> None:
|
|
required = ["OpenAPI required scenario"]
|
|
reports = [
|
|
{"suites": []},
|
|
{"suites": [{"specs": [{"title": "another scenario", "tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}]}]}]},
|
|
{"suites": [{"specs": [{"title": required[0], "tests": [{"status": "skipped", "results": []}]}]}]},
|
|
{"suites": [{"specs": [{"title": required[0], "tests": [{"status": "flaky", "results": [{"status": "failed", "retry": 0}, {"status": "passed", "retry": 1}]}]}]}]},
|
|
]
|
|
for report in reports:
|
|
with self.subTest(report=report):
|
|
result, output, temporary = self.run_playwright(report, required)
|
|
self.addCleanup(temporary.cleanup)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
candidate = json.loads(output.read_text(encoding="utf-8"))
|
|
self.assertEqual(candidate["execution_verdict"], "fail")
|
|
self.assertFalse(candidate["accepted"])
|
|
|
|
def test_required_scope_ignores_unrelated_flaky_tests(self) -> None:
|
|
required = "OpenAPI required scenario"
|
|
report = {
|
|
"suites": [
|
|
{
|
|
"specs": [
|
|
{
|
|
"title": required,
|
|
"tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}],
|
|
},
|
|
{
|
|
"title": "Unrelated wizard scenario",
|
|
"tests": [
|
|
{
|
|
"status": "flaky",
|
|
"results": [
|
|
{"status": "failed", "retry": 0},
|
|
{"status": "passed", "retry": 1},
|
|
],
|
|
}
|
|
],
|
|
},
|
|
{"title": "Unrelated failure", "tests": [{"status": "unexpected", "results": [{"status": "failed", "retry": 0}]}]},
|
|
{"title": "Unrelated skip", "tests": [{"status": "skipped", "results": []}]},
|
|
{"title": "Unrelated malformed", "tests": [{"status": "expected", "results": None}]},
|
|
]
|
|
}
|
|
]
|
|
}
|
|
result, output, temporary = self.run_playwright(report, [required])
|
|
self.addCleanup(temporary.cleanup)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
candidate = json.loads(output.read_text(encoding="utf-8"))
|
|
self.assertEqual(candidate["execution_verdict"], "pass")
|
|
self.assertTrue(candidate["accepted"])
|
|
self.assertEqual(candidate["summary"], {"passed": 1, "failed": 0, "flaky": 0, "skipped": 0, "not_run": 0})
|
|
|
|
def test_final_failure_after_retry_is_failed_not_flaky(self) -> None:
|
|
for status in ("unexpected", "flaky"):
|
|
with self.subTest(status=status):
|
|
report = {
|
|
"suites": [
|
|
{
|
|
"specs": [
|
|
{
|
|
"tests": [
|
|
{
|
|
"status": status,
|
|
"results": [
|
|
{"status": "failed", "retry": 0},
|
|
{"status": "failed", "retry": 1},
|
|
],
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
result, output, temporary = self.run_playwright(report)
|
|
self.addCleanup(temporary.cleanup)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
candidate = json.loads(output.read_text(encoding="utf-8"))
|
|
self.assertEqual(candidate["execution_verdict"], "fail")
|
|
self.assertEqual(candidate["summary"]["failed"], 1)
|
|
self.assertEqual(candidate["summary"]["flaky"], 0)
|
|
|
|
def test_raw_report_content_never_reaches_candidate_or_error(self) -> None:
|
|
canary = "Bearer secret-canary /home/private/workspace https://private.invalid?q=secret"
|
|
report = {"suites": [], "errors": [{"message": canary}], "stdout": [canary]}
|
|
result, output, temporary = self.run_playwright(report)
|
|
self.addCleanup(temporary.cleanup)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(json.loads(output.read_text(encoding="utf-8"))["execution_verdict"], "fail")
|
|
self.assertNotIn(canary, output.read_text(encoding="utf-8"))
|
|
self.assertNotIn(canary, result.stderr)
|
|
|
|
def test_report_level_error_and_malformed_result_never_pass(self) -> None:
|
|
reports = [
|
|
{"errors": [{"message": "fatal"}], "suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed"}]}]}]}]},
|
|
{"suites": [{"specs": [{"tests": [{"status": "expected", "results": ["not-an-object"]}]}]}]},
|
|
{"suites": [{"specs": [{"tests": [{"status": "expected", "results": None}]}]}]},
|
|
{"suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed", "retry": True}]}]}]}]},
|
|
]
|
|
for report in reports:
|
|
with self.subTest(report=report):
|
|
result, output, temporary = self.run_playwright(report)
|
|
self.addCleanup(temporary.cleanup)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertFalse(json.loads(output.read_text(encoding="utf-8"))["accepted"])
|
|
|
|
def test_oversized_report_fails_without_traceback(self) -> None:
|
|
temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
root = Path(temporary.name)
|
|
report = root / "report.json"
|
|
report.write_bytes(b" " * (4 * 1024 * 1024 + 1))
|
|
result = subprocess.run(
|
|
["python3", str(COLLECTOR), "playwright", "--report", str(report), "--output", str(root / "out.json"), "--source-revision", "0" * 40, "--environment-class", "community-test", "--flow-id", "ui-operations"],
|
|
text=True, capture_output=True, check=False,
|
|
)
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("INPUT_TOO_LARGE", result.stderr)
|
|
self.assertNotIn("Traceback", result.stderr)
|
|
|
|
def test_allowlisted_command_report_derives_exit_verdict(self) -> None:
|
|
temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
root = Path(temporary.name)
|
|
report = root / "command.json"
|
|
output = root / "candidate.json"
|
|
report.write_text(json.dumps({"command_id": "rust-admin-integration", "exit_code": 7, "timed_out": False, "skipped": 0}), encoding="utf-8")
|
|
result = subprocess.run(
|
|
["python3", str(COLLECTOR), "command-report", "--report", str(report), "--output", str(output), "--source-revision", "0" * 40, "--environment-class", "community-test", "--flow-id", "api-operation-test-run"],
|
|
text=True, capture_output=True, check=False,
|
|
)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
candidate = json.loads(output.read_text(encoding="utf-8"))
|
|
self.assertEqual(candidate["execution_verdict"], "fail")
|
|
self.assertFalse(candidate["accepted"])
|
|
|
|
def test_unknown_command_report_is_rejected(self) -> None:
|
|
temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
root = Path(temporary.name)
|
|
report = root / "command.json"
|
|
report.write_text(json.dumps({"command_id": "arbitrary-shell", "exit_code": 0, "timed_out": False, "skipped": 0}), encoding="utf-8")
|
|
result = subprocess.run(
|
|
["python3", str(COLLECTOR), "command-report", "--report", str(report), "--output", str(root / "candidate.json"), "--source-revision", "0" * 40, "--environment-class", "community-test", "--flow-id", "api-operation-test-run"],
|
|
text=True, capture_output=True, check=False,
|
|
)
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("UNKNOWN_COMMAND", result.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|