import hashlib import io import json import os import re import shutil import subprocess import tarfile import tempfile import unittest from datetime import datetime, timedelta, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[2] WRAPPER = ROOT / "scripts" / "scan-images.sh" CHECKER = ROOT / "scripts" / "check-image-scan.py" VULN_ID = "sha256:" + "1" * 64 CLEAN_ID = "sha256:" + "2" * 64 VULN_TAG = "crank/vuln:ci" CLEAN_TAG = "crank/clean:ci" FAKE_CURL = """#!/usr/bin/env python3 import os import sys args = sys.argv[1:] if os.environ.get("FAKE_CURL_FAIL") == "1": sys.stderr.write("curl: (7) failed to connect\\n") sys.exit(7) output = None for index, value in enumerate(args): if value in ("--output", "-o"): output = args[index + 1] if output is None: sys.stderr.write("curl: missing --output\\n") sys.exit(2) with open(os.environ["FAKE_CURL_ARCHIVE"], "rb") as source: payload = source.read() with open(output, "wb") as target: target.write(payload) sys.exit(0) """ FAKE_DOCKER = """#!/usr/bin/env python3 import json import os import sys args = sys.argv[1:] with open(os.environ["FAKE_DOCKER_LOG"], "a", encoding="utf-8") as stream: stream.write(json.dumps(args) + "\\n") if args[:2] == ["image", "inspect"]: with open(os.environ["FAKE_IMAGE_MAP"], encoding="utf-8") as source: mapping = json.load(source) entry = mapping.get(args[-1]) if entry is None: sys.stderr.write("docker: no such image " + args[-1] + "\\n") sys.exit(1) sys.stdout.write(entry["id"] + "\\n") sys.exit(0) sys.stderr.write("docker: unexpected command\\n") sys.exit(2) """ # Fake Trivy shipped inside the stub scanner archive. It records every # invocation and any inherited TRIVY_* variable so the test can prove the # wrapper both pinned the flags and scrubbed the environment. FAKE_TRIVY = """#!/usr/bin/env python3 import json import os import sys from datetime import datetime, timedelta, timezone args = sys.argv[1:] log = os.environ.get("FAKE_TRIVY_LOG") if log: record = { "args": args, "trivy_env": sorted(name for name in os.environ if name.startswith("TRIVY_")), } with open(log, "a", encoding="utf-8") as stream: stream.write(json.dumps(record) + "\\n") if "--version" in args: if os.environ.get("FAKE_TRIVY_VERSION_FAIL"): sys.exit(7) sys.stdout.write("Version: 0.70.0\\n") sys.exit(0) if "--download-db-only" in args: cache = args[args.index("--cache-dir") + 1] db_dir = os.path.join(cache, "db") os.makedirs(db_dir, exist_ok=True) with open(os.path.join(db_dir, "trivy.db"), "w", encoding="utf-8") as stream: stream.write("fixture-db\\n") now = datetime.now(timezone.utc) metadata = { "Version": 2, "UpdatedAt": now.isoformat().replace("+00:00", "Z"), "NextUpdate": (now + timedelta(hours=24)).isoformat().replace("+00:00", "Z"), } with open(os.path.join(db_dir, "metadata.json"), "w", encoding="utf-8") as stream: json.dump(metadata, stream) sys.exit(0) target = args[-1] if os.environ.get("FAKE_TRIVY_NO_OUTPUT_ID") == target: sys.exit(0) if os.environ.get("FAKE_TRIVY_FAIL_ID") == target: sys.stderr.write("trivy: fixture scan failure for " + target + "\\n") sys.exit(1) output = args[args.index("--output") + 1] source = os.path.join(os.environ["FAKE_REPORT_DIR"], target.replace(":", "_") + ".json") if not os.path.exists(source): sys.stderr.write("trivy: no report fixture for " + target + "\\n") sys.exit(1) with open(source, encoding="utf-8") as stream: report = json.load(stream) report["Metadata"]["ImageID"] = target with open(output, "w", encoding="utf-8") as stream: json.dump(report, stream) sys.exit(0) """ def build_archive(script: str) -> bytes: buffer = io.BytesIO() with tarfile.open(fileobj=buffer, mode="w:gz") as archive: data = script.encode("utf-8") info = tarfile.TarInfo("trivy") info.size = len(data) info.mode = 0o755 info.type = tarfile.REGTYPE archive.addfile(info, io.BytesIO(data)) return buffer.getvalue() def alpine_report(image_id: str, vulnerabilities=None) -> dict: result = { "Target": "fixture:latest (alpine 3.24)", "Class": "os-pkgs", "Type": "alpine", "Packages": [ {"ID": "musl@1.2.5-r0", "Name": "musl", "Version": "1.2.5-r0"} ], } if vulnerabilities is not None: result["Vulnerabilities"] = vulnerabilities return { "SchemaVersion": 2, "ArtifactName": "fixture:latest", "ArtifactType": "container_image", "Metadata": { "ImageID": image_id, "OS": {"Family": "alpine", "Name": "3.24"}, }, "Results": [result], } def unfixed_high() -> dict: return { "VulnerabilityID": "CVE-2024-0001", "PkgName": "musl", "InstalledVersion": "1.2.5-r0", "Severity": "HIGH", "Status": "affected", "Title": "fixture unfixed high", } class ScanImagesWrapperTests(unittest.TestCase): def run_scan( self, *, images=(VULN_TAG, CLEAN_TAG), trivy_fail_id=None, curl_fail=False, extra_env=None, workdir_files=None, run_in_workdir=False, repeat_without_output=False, ): temporary = tempfile.TemporaryDirectory() self.addCleanup(temporary.cleanup) root = Path(temporary.name) bin_dir = root / "bin" bin_dir.mkdir() for name, contents in (("curl", FAKE_CURL), ("docker", FAKE_DOCKER)): path = bin_dir / name path.write_text(contents, encoding="utf-8") path.chmod(0o755) wrappers = root / "wrappers" wrappers.mkdir() fixture_archive = build_archive(FAKE_TRIVY) production = WRAPPER.read_text(encoding="utf-8") fixture_hash = hashlib.sha256(fixture_archive).hexdigest() # Replace only the pinned archive hash in the test copy; the production # wrapper is never modified and there is no bypass environment variable. patched, replacements = re.subn( r'expected_sha256="[0-9a-f]{64}"', f'expected_sha256="{fixture_hash}"', production, count=1, ) self.assertEqual(replacements, 1) self.assertIn('expected_sha256="8b4376d5d6befe5c24d503f10ff136d9e0c49f9127a4279fd110b727929a5aa9"', production) wrapper_copy = wrappers / "scan-images.sh" wrapper_copy.write_text(patched, encoding="utf-8") wrapper_copy.chmod(0o755) shutil.copy2(CHECKER, wrappers / "check-image-scan.py") archive_path = root / "trivy-fixture.tar.gz" archive_path.write_bytes(fixture_archive) reports = root / "reports" reports.mkdir() (reports / (VULN_ID.replace(":", "_") + ".json")).write_text( json.dumps(alpine_report(VULN_ID, [unfixed_high()])), encoding="utf-8" ) (reports / (CLEAN_ID.replace(":", "_") + ".json")).write_text( json.dumps(alpine_report(CLEAN_ID)), encoding="utf-8" ) image_map = root / "images.json" image_map.write_text( json.dumps( { VULN_TAG: {"id": VULN_ID}, CLEAN_TAG: {"id": CLEAN_ID}, } ), encoding="utf-8", ) evidence = root / "evidence" docker_log = root / "docker.jsonl" trivy_log = root / "trivy.jsonl" for name, contents in (workdir_files or {}).items(): (root / name).write_text(contents, encoding="utf-8") env = os.environ.copy() env.update( { "PATH": f"{bin_dir}:{env['PATH']}", "CRANK_IMAGE_SCAN_OUTPUT_DIR": str(evidence), "FAKE_CURL_ARCHIVE": str(archive_path), "FAKE_IMAGE_MAP": str(image_map), "FAKE_REPORT_DIR": str(reports), "FAKE_DOCKER_LOG": str(docker_log), "FAKE_TRIVY_LOG": str(trivy_log), } ) if trivy_fail_id is not None: env["FAKE_TRIVY_FAIL_ID"] = trivy_fail_id if curl_fail: env["FAKE_CURL_FAIL"] = "1" if extra_env: env.update(extra_env) result = subprocess.run( ["sh", str(wrapper_copy), *images], text=True, capture_output=True, env=env, cwd=str(root) if run_in_workdir else None, check=False, ) if repeat_without_output: self.assertEqual(result.returncode, 0, result.stderr) env["FAKE_TRIVY_NO_OUTPUT_ID"] = CLEAN_ID result = subprocess.run(["sh", str(wrapper_copy), *images], text=True, capture_output=True, env=env, cwd=str(root) if run_in_workdir else None, check=False) docker_calls = [] if docker_log.exists(): docker_calls = [ json.loads(line) for line in docker_log.read_text(encoding="utf-8").splitlines() if line.strip() ] trivy_calls = [] if trivy_log.exists(): trivy_calls = [ json.loads(line) for line in trivy_log.read_text(encoding="utf-8").splitlines() if line.strip() ] receipt = None runs = sorted(evidence.glob("run.*"), key=lambda path: path.stat().st_mtime_ns) if runs: evidence = runs[-1] receipt_path = evidence / "receipt.json" if receipt_path.exists(): receipt = json.loads(receipt_path.read_text(encoding="utf-8")) return result, root, evidence, receipt, docker_calls, trivy_calls @staticmethod def scan_calls(trivy_calls): return [call for call in trivy_calls if call["args"][:1] == ["image"] and "--output" in call["args"]] @staticmethod def target_of(call): return call["args"][-1] def test_unfixed_high_blocks_overall_but_both_images_scanned(self): result, _, evidence, receipt, docker_calls, trivy_calls = self.run_scan() self.assertEqual(result.returncode, 1, result.stderr) scans = self.scan_calls(trivy_calls) self.assertEqual({self.target_of(call) for call in scans}, {VULN_ID, CLEAN_ID}) self.assertEqual( {call[-1] for call in docker_calls if call[:2] == ["image", "inspect"]}, {VULN_TAG, CLEAN_TAG}, ) self.assertIsNotNone(receipt) self.assertEqual(receipt["overall"]["exit_code"], 1) self.assertEqual(receipt["overall"]["status"], "blocked") self.assertEqual(len(receipt["images"]), 2) self.assertEqual({entry["tag"] for entry in receipt["images"]}, {VULN_TAG, CLEAN_TAG}) self.assertIn(VULN_ID, {entry["image_id"] for entry in receipt["images"]}) by_tag = {entry["tag"]: entry for entry in receipt["images"]} for entry in receipt["images"]: self.assertTrue(Path(entry["raw"]).is_file(), entry["raw"]) self.assertTrue(Path(entry["decision"]).is_file(), entry["decision"]) self.assertTrue(Path(entry["scan_log"]).is_file(), entry["scan_log"]) self.assertEqual(by_tag[VULN_TAG]["status"], "blocked") self.assertEqual(by_tag[CLEAN_TAG]["status"], "pass") vuln_raw = json.loads(Path(by_tag[VULN_TAG]["raw"]).read_text(encoding="utf-8")) clean_raw = json.loads(Path(by_tag[CLEAN_TAG]["raw"]).read_text(encoding="utf-8")) self.assertEqual(vuln_raw["Metadata"]["ImageID"], VULN_ID) self.assertEqual(clean_raw["Metadata"]["ImageID"], CLEAN_ID) vuln_decision = json.loads(Path(by_tag[VULN_TAG]["decision"]).read_text(encoding="utf-8")) clean_decision = json.loads(Path(by_tag[CLEAN_TAG]["decision"]).read_text(encoding="utf-8")) self.assertEqual(vuln_decision["status"], "blocked") self.assertEqual(vuln_decision["blocking"][0]["vulnerability_id"], "CVE-2024-0001") self.assertEqual(clean_decision["status"], "pass") def test_pinned_flags_survive_hidden_trivy_overrides(self): hostile = { "TRIVY_SEVERITY": "LOW", "TRIVY_IGNORE_UNFIXED": "true", "TRIVY_CONFIG": "/dev/null", "TRIVY_SKIP_FILES": "usr/bin/infocmp", "TRIVY_VEX": "vex.json", "TRIVY_IGNORE_POLICY": "policy.rego", } result, _, _, _, _, trivy_calls = self.run_scan( extra_env=hostile, workdir_files={"trivy.yaml": "severity: LOW\nignore-unfixed: true\n"}, run_in_workdir=True, ) self.assertEqual(result.returncode, 1, result.stderr) for call in trivy_calls: args = call["args"] self.assertIn("--config", args) self.assertEqual(args[args.index("--config") + 1], "/dev/null") scans = self.scan_calls(trivy_calls) self.assertTrue(scans) for call in scans: args = call["args"] self.assertEqual(call["trivy_env"], []) self.assertIn("--image-src", args) self.assertEqual(args[args.index("--image-src") + 1], "docker") self.assertEqual(args[args.index("--config") + 1], "/dev/null") self.assertEqual(args[args.index("--ignorefile") + 1], "/dev/null") self.assertIn("--ignore-unfixed=false", args) self.assertIn("--list-all-pkgs=true", args) self.assertIn("--skip-db-update", args) self.assertIn("--skip-version-check", args) self.assertIn("--no-progress", args) self.assertEqual(args[args.index("--exit-code") + 1], "0") self.assertEqual(args[args.index("--scanners") + 1], "vuln") self.assertEqual(args[args.index("--format") + 1], "json") self.assertEqual( args[args.index("--severity") + 1], "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL", ) def test_scan_error_on_first_image_does_not_stop_the_second(self): result, _, _, receipt, _, trivy_calls = self.run_scan(trivy_fail_id=VULN_ID) self.assertEqual(result.returncode, 2, result.stderr) scans = self.scan_calls(trivy_calls) self.assertEqual({self.target_of(call) for call in scans}, {VULN_ID, CLEAN_ID}) self.assertIsNotNone(receipt) self.assertEqual(receipt["overall"]["exit_code"], 2) self.assertEqual(receipt["overall"]["status"], "error") by_tag = {entry["tag"]: entry for entry in receipt["images"]} self.assertEqual(by_tag[VULN_TAG]["status"], "error") self.assertEqual(by_tag[CLEAN_TAG]["status"], "pass") clean_decision = json.loads( Path(by_tag[CLEAN_TAG]["decision"]).read_text(encoding="utf-8") ) self.assertEqual(clean_decision["status"], "pass") vuln_decision = json.loads( Path(by_tag[VULN_TAG]["decision"]).read_text(encoding="utf-8") ) self.assertEqual(vuln_decision["status"], "error") def test_scanner_version_failure_is_a_bootstrap_error(self): result, _, _, receipt, _, _ = self.run_scan(extra_env={"FAKE_TRIVY_VERSION_FAIL": "1"}) self.assertEqual(result.returncode, 2, result.stderr) self.assertEqual(receipt["overall"]["status"], "error") def test_rerun_cannot_reuse_an_old_successful_raw_report(self): result, _, evidence, receipt, _, _ = self.run_scan(images=(CLEAN_TAG,), repeat_without_output=True) self.assertEqual(result.returncode, 2, result.stderr) self.assertEqual(receipt["overall"]["status"], "error") self.assertEqual(receipt["images"][0]["status"], "error") def test_bootstrap_failure_writes_error_receipt_for_each_image(self): result, _, _, receipt, docker_calls, trivy_calls = self.run_scan(curl_fail=True) self.assertEqual(result.returncode, 2, result.stderr) self.assertIsNotNone(receipt) self.assertEqual(receipt["overall"]["status"], "error") self.assertEqual(receipt["overall"]["exit_code"], 2) self.assertEqual(len(receipt["images"]), 2) self.assertTrue(all(entry["status"] == "error" for entry in receipt["images"])) self.assertEqual({entry["tag"] for entry in receipt["images"]}, {VULN_TAG, CLEAN_TAG}) self.assertEqual(docker_calls, []) self.assertEqual(trivy_calls, []) def test_unknown_image_is_error_and_is_never_scanned(self): missing = "crank/missing:ci" result, _, _, receipt, _, trivy_calls = self.run_scan(images=(missing, CLEAN_TAG)) self.assertEqual(result.returncode, 2, result.stderr) by_tag = {entry["tag"]: entry for entry in receipt["images"]} self.assertEqual(by_tag[missing]["status"], "error") self.assertEqual(by_tag[CLEAN_TAG]["status"], "pass") self.assertEqual([self.target_of(call) for call in self.scan_calls(trivy_calls)], [CLEAN_ID]) decision = json.loads(Path(by_tag[missing]["decision"]).read_text()) self.assertEqual(decision["status"], "error") def test_download_error_log_is_retained(self): result, _, evidence, receipt, _, _ = self.run_scan(curl_fail=True) self.assertEqual(result.returncode, 2, result.stderr) self.assertEqual(receipt["overall"]["status"], "error") self.assertIn("failed to connect", (evidence / "scanner-download.log").read_text()) def test_evidence_kept_even_when_gate_is_red(self): _, _, evidence, receipt, _, _ = self.run_scan() self.assertTrue((evidence / "receipt.json").is_file()) self.assertTrue((evidence / "scanner.json").is_file()) self.assertTrue((evidence / "db" / "metadata.json").is_file()) scanner = json.loads((evidence / "scanner.json").read_text(encoding="utf-8")) self.assertEqual(scanner["expected_version"], "0.70.0") self.assertEqual(scanner["policy"]["ignore_unfixed"], False) self.assertIsNotNone(receipt) if __name__ == "__main__": unittest.main()