feat: harden community production foundation through story 1.5

tests/unit/test_collect_capability_baseline.py

@@ -0,0 +1,121 @@
+import json
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[2]
+COLLECTOR = ROOT / "scripts" / "collect-capability-baseline.py"
+
+
+class CapabilityBaselineCollectorTests(unittest.TestCase):
+    def run_playwright(self, report: object) -> tuple[subprocess.CompletedProcess[str], Path, tempfile.TemporaryDirectory[str]]:
+        temporary = tempfile.TemporaryDirectory()
+        root = Path(temporary.name)
+        report_path = root / "report.json"
+        output_path = root / "candidate.json"
+        report_path.write_text(json.dumps(report), encoding="utf-8")
+        result = subprocess.run(
+            ["python3", str(COLLECTOR), "playwright", "--report", str(report_path), "--output", str(output_path), "--source-revision", "0" * 40, "--environment-class", "community-test", "--flow-id", "ui-operations"],
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        return result, output_path, temporary
+
+    def test_stable_playwright_pass_is_collected(self) -> None:
+        report = {"suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed", "retry": 0}]}]}]}]}
+        result, output, temporary = self.run_playwright(report)
+        self.addCleanup(temporary.cleanup)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        candidate = json.loads(output.read_text(encoding="utf-8"))
+        self.assertEqual(candidate["execution_verdict"], "pass")
+        self.assertTrue(candidate["accepted"])
+        self.assertEqual(candidate["evidence_mode"], "automated")
+        self.assertRegex(candidate["source_report_sha256"], "^[0-9a-f]{64}$")
+
+    def test_retry_pass_is_flaky_and_not_accepted(self) -> None:
+        report = {"suites": [{"specs": [{"tests": [{"status": "flaky", "results": [{"status": "failed", "retry": 0}, {"status": "passed", "retry": 1}]}]}]}]}
+        result, output, temporary = self.run_playwright(report)
+        self.addCleanup(temporary.cleanup)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        candidate = json.loads(output.read_text(encoding="utf-8"))
+        self.assertEqual(candidate["execution_verdict"], "flaky")
+        self.assertFalse(candidate["accepted"])
+
+    def test_skipped_and_missing_results_are_not_pass(self) -> None:
+        report = {"suites": [{"specs": [{"tests": [{"status": "skipped", "results": []}]}]}]}
+        result, output, temporary = self.run_playwright(report)
+        self.addCleanup(temporary.cleanup)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(json.loads(output.read_text(encoding="utf-8"))["execution_verdict"], "skipped")
+
+    def test_raw_report_content_never_reaches_candidate_or_error(self) -> None:
+        canary = "Bearer secret-canary /home/private/workspace https://private.invalid?q=secret"
+        report = {"suites": [], "errors": [{"message": canary}], "stdout": [canary]}
+        result, output, temporary = self.run_playwright(report)
+        self.addCleanup(temporary.cleanup)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(json.loads(output.read_text(encoding="utf-8"))["execution_verdict"], "fail")
+        self.assertNotIn(canary, output.read_text(encoding="utf-8"))
+        self.assertNotIn(canary, result.stderr)
+
+    def test_report_level_error_and_malformed_result_never_pass(self) -> None:
+        reports = [
+            {"errors": [{"message": "fatal"}], "suites": [{"specs": [{"tests": [{"status": "expected", "results": [{"status": "passed"}]}]}]}]},
+            {"suites": [{"specs": [{"tests": [{"status": "expected", "results": ["not-an-object"]}]}]}]},
+        ]
+        for report in reports:
+            with self.subTest(report=report):
+                result, output, temporary = self.run_playwright(report)
+                self.addCleanup(temporary.cleanup)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertFalse(json.loads(output.read_text(encoding="utf-8"))["accepted"])
+
+    def test_oversized_report_fails_without_traceback(self) -> None:
+        temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(temporary.cleanup)
+        root = Path(temporary.name)
+        report = root / "report.json"
+        report.write_bytes(b" " * (4 * 1024 * 1024 + 1))
+        result = subprocess.run(
+            ["python3", str(COLLECTOR), "playwright", "--report", str(report), "--output", str(root / "out.json"), "--source-revision", "0" * 40, "--environment-class", "community-test", "--flow-id", "ui-operations"],
+            text=True, capture_output=True, check=False,
+        )
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("INPUT_TOO_LARGE", result.stderr)
+        self.assertNotIn("Traceback", result.stderr)
+
+    def test_allowlisted_command_report_derives_exit_verdict(self) -> None:
+        temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(temporary.cleanup)
+        root = Path(temporary.name)
+        report = root / "command.json"
+        output = root / "candidate.json"
+        report.write_text(json.dumps({"command_id": "rust-admin-integration", "exit_code": 7, "timed_out": False, "skipped": 0}), encoding="utf-8")
+        result = subprocess.run(
+            ["python3", str(COLLECTOR), "command-report", "--report", str(report), "--output", str(output), "--source-revision", "0" * 40, "--environment-class", "community-test", "--flow-id", "api-operation-test-run"],
+            text=True, capture_output=True, check=False,
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        candidate = json.loads(output.read_text(encoding="utf-8"))
+        self.assertEqual(candidate["execution_verdict"], "fail")
+        self.assertFalse(candidate["accepted"])
+
+    def test_unknown_command_report_is_rejected(self) -> None:
+        temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(temporary.cleanup)
+        root = Path(temporary.name)
+        report = root / "command.json"
+        report.write_text(json.dumps({"command_id": "arbitrary-shell", "exit_code": 0, "timed_out": False, "skipped": 0}), encoding="utf-8")
+        result = subprocess.run(
+            ["python3", str(COLLECTOR), "command-report", "--report", str(report), "--output", str(root / "candidate.json"), "--source-revision", "0" * 40, "--environment-class", "community-test", "--flow-id", "api-operation-test-run"],
+            text=True, capture_output=True, check=False,
+        )
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("UNKNOWN_COMMAND", result.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main()