Files
crank/tests/unit/test_scan_images.py

565 lines
24 KiB
Python

import hashlib
import io
import json
import os
import re
import shutil
import subprocess
import tarfile
import tempfile
import unittest
from datetime import datetime, timedelta, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
WRAPPER = ROOT / "scripts" / "scan-images.sh"
CHECKER = ROOT / "scripts" / "check-image-scan.py"
VULN_ID = "sha256:" + "1" * 64
CLEAN_ID = "sha256:" + "2" * 64
VULN_TAG = "crank/vuln:ci"
CLEAN_TAG = "crank/clean:ci"
FAKE_CURL = """#!/usr/bin/env python3
import os
import sys
args = sys.argv[1:]
url = next((value for value in args if value.startswith("https://")), "")
api_download = url == "https://api.github.com/repos/aquasecurity/trivy/releases/assets/398381981"
failed = os.environ.get("FAKE_CURL_FAIL") == "1"
failed = failed or (not api_download and os.environ.get("FAKE_CURL_DIRECT_FAIL") == "1")
failed = failed or (api_download and os.environ.get("FAKE_CURL_API_FAIL") == "1")
if failed:
sys.stderr.write("curl: (22) fixture download failure for " + url + "\\n")
sys.exit(7)
if api_download:
if "--header" not in args or args[args.index("--header") + 1] != "Accept: application/octet-stream":
sys.stderr.write("curl: missing octet-stream Accept header\\n")
sys.exit(2)
output = None
for index, value in enumerate(args):
if value in ("--output", "-o"):
output = args[index + 1]
if output is None:
sys.stderr.write("curl: missing --output\\n")
sys.exit(2)
archive_path = os.environ.get("FAKE_CURL_API_ARCHIVE", os.environ["FAKE_CURL_ARCHIVE"]) if api_download else os.environ["FAKE_CURL_ARCHIVE"]
with open(archive_path, "rb") as source:
payload = source.read()
with open(output, "wb") as target:
target.write(payload)
sys.exit(0)
"""
FAKE_DOCKER = """#!/usr/bin/env python3
import json
import os
import sys
args = sys.argv[1:]
with open(os.environ["FAKE_DOCKER_LOG"], "a", encoding="utf-8") as stream:
stream.write(json.dumps(args) + "\\n")
if args[:2] == ["image", "inspect"]:
with open(os.environ["FAKE_IMAGE_MAP"], encoding="utf-8") as source:
mapping = json.load(source)
entry = mapping.get(args[-1])
if entry is None:
sys.stderr.write("docker: no such image " + args[-1] + "\\n")
sys.exit(1)
sys.stdout.write(entry["id"] + "\\n")
sys.exit(0)
sys.stderr.write("docker: unexpected command\\n")
sys.exit(2)
"""
# Fake Trivy shipped inside the stub scanner archive. It records every
# invocation and any inherited TRIVY_* variable so the test can prove the
# wrapper both pinned the flags and scrubbed the environment.
FAKE_TRIVY = """#!/usr/bin/env python3
import json
import os
import sys
from datetime import datetime, timedelta, timezone
args = sys.argv[1:]
log = os.environ.get("FAKE_TRIVY_LOG")
if log:
record = {
"args": args,
"trivy_env": sorted(name for name in os.environ if name.startswith("TRIVY_")),
}
with open(log, "a", encoding="utf-8") as stream:
stream.write(json.dumps(record) + "\\n")
if "--version" in args:
if os.environ.get("FAKE_TRIVY_VERSION_FAIL"):
sys.exit(7)
sys.stdout.write("Version: 0.70.0\\n")
sys.exit(0)
if "--download-db-only" in args:
cache = args[args.index("--cache-dir") + 1]
db_dir = os.path.join(cache, "db")
os.makedirs(db_dir, exist_ok=True)
with open(os.path.join(db_dir, "trivy.db"), "w", encoding="utf-8") as stream:
stream.write("fixture-db\\n")
now = datetime.now(timezone.utc)
metadata = {
"Version": 2,
"UpdatedAt": now.isoformat().replace("+00:00", "Z"),
"NextUpdate": (now + timedelta(hours=24)).isoformat().replace("+00:00", "Z"),
}
with open(os.path.join(db_dir, "metadata.json"), "w", encoding="utf-8") as stream:
json.dump(metadata, stream)
sys.exit(0)
target = args[-1]
if os.environ.get("FAKE_TRIVY_NO_OUTPUT_ID") == target:
sys.exit(0)
if os.environ.get("FAKE_TRIVY_FAIL_ID") == target:
sys.stderr.write("trivy: fixture scan failure for " + target + "\\n")
sys.exit(1)
output = args[args.index("--output") + 1]
source = os.path.join(os.environ["FAKE_REPORT_DIR"], target.replace(":", "_") + ".json")
if not os.path.exists(source):
sys.stderr.write("trivy: no report fixture for " + target + "\\n")
sys.exit(1)
with open(source, encoding="utf-8") as stream:
report = json.load(stream)
report["Metadata"]["ImageID"] = target
with open(output, "w", encoding="utf-8") as stream:
json.dump(report, stream)
sys.exit(0)
"""
def build_archive(script: str) -> bytes:
buffer = io.BytesIO()
with tarfile.open(fileobj=buffer, mode="w:gz") as archive:
data = script.encode("utf-8")
info = tarfile.TarInfo("trivy")
info.size = len(data)
info.mode = 0o755
info.type = tarfile.REGTYPE
archive.addfile(info, io.BytesIO(data))
return buffer.getvalue()
def alpine_report(image_id: str, vulnerabilities=None) -> dict:
result = {
"Target": "fixture:latest (alpine 3.24)",
"Class": "os-pkgs",
"Type": "alpine",
"Packages": [
{"ID": "musl@1.2.5-r0", "Name": "musl", "Version": "1.2.5-r0"}
],
}
if vulnerabilities is not None:
result["Vulnerabilities"] = vulnerabilities
return {
"SchemaVersion": 2,
"ArtifactName": "fixture:latest",
"ArtifactType": "container_image",
"Metadata": {
"ImageID": image_id,
"OS": {"Family": "alpine", "Name": "3.24"},
},
"Results": [result],
}
def unfixed_high() -> dict:
return {
"VulnerabilityID": "CVE-2024-0001",
"PkgName": "musl",
"InstalledVersion": "1.2.5-r0",
"Severity": "HIGH",
"Status": "affected",
"Title": "fixture unfixed high",
}
class ScanImagesWrapperTests(unittest.TestCase):
def run_scan(
self,
*,
images=(VULN_TAG, CLEAN_TAG),
trivy_fail_id=None,
curl_fail=False,
curl_direct_fail=False,
curl_api_fail=False,
api_checksum_mismatch=False,
cache_archive=None,
extra_env=None,
workdir_files=None,
run_in_workdir=False,
repeat_without_output=False,
):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
root = Path(temporary.name)
bin_dir = root / "bin"
bin_dir.mkdir()
for name, contents in (("curl", FAKE_CURL), ("docker", FAKE_DOCKER)):
path = bin_dir / name
path.write_text(contents, encoding="utf-8")
path.chmod(0o755)
wrappers = root / "wrappers"
wrappers.mkdir()
fixture_archive = build_archive(FAKE_TRIVY)
production = WRAPPER.read_text(encoding="utf-8")
fixture_hash = hashlib.sha256(fixture_archive).hexdigest()
# Replace only the pinned archive hash in the test copy; the production
# wrapper is never modified and there is no bypass environment variable.
patched, replacements = re.subn(
r'expected_sha256="[0-9a-f]{64}"',
f'expected_sha256="{fixture_hash}"',
production,
count=1,
)
self.assertEqual(replacements, 1)
self.assertIn('expected_sha256="8b4376d5d6befe5c24d503f10ff136d9e0c49f9127a4279fd110b727929a5aa9"', production)
wrapper_copy = wrappers / "scan-images.sh"
wrapper_copy.write_text(patched, encoding="utf-8")
wrapper_copy.chmod(0o755)
shutil.copy2(CHECKER, wrappers / "check-image-scan.py")
archive_path = root / "trivy-fixture.tar.gz"
archive_path.write_bytes(fixture_archive)
api_archive_path = root / "trivy-api-fixture.tar.gz"
api_archive_path.write_bytes(fixture_archive + b"tampered" if api_checksum_mismatch else fixture_archive)
tool_cache = root / "runner-tool-cache"
if cache_archive is not None:
cached_path = tool_cache / "trivy" / "0.70.0" / "trivy_0.70.0_Linux-64bit.tar.gz"
cached_path.parent.mkdir(parents=True)
cached_path.write_bytes(fixture_archive if cache_archive == "valid" else b"invalid archive")
reports = root / "reports"
reports.mkdir()
(reports / (VULN_ID.replace(":", "_") + ".json")).write_text(
json.dumps(alpine_report(VULN_ID, [unfixed_high()])), encoding="utf-8"
)
(reports / (CLEAN_ID.replace(":", "_") + ".json")).write_text(
json.dumps(alpine_report(CLEAN_ID)), encoding="utf-8"
)
image_map = root / "images.json"
image_map.write_text(
json.dumps(
{
VULN_TAG: {"id": VULN_ID},
CLEAN_TAG: {"id": CLEAN_ID},
}
),
encoding="utf-8",
)
evidence = root / "evidence"
docker_log = root / "docker.jsonl"
trivy_log = root / "trivy.jsonl"
for name, contents in (workdir_files or {}).items():
(root / name).write_text(contents, encoding="utf-8")
env = os.environ.copy()
env.update(
{
"PATH": f"{bin_dir}:{env['PATH']}",
"RUNNER_TOOL_CACHE": str(tool_cache),
"CRANK_IMAGE_SCAN_OUTPUT_DIR": str(evidence),
"FAKE_CURL_ARCHIVE": str(archive_path),
"FAKE_CURL_API_ARCHIVE": str(api_archive_path),
"FAKE_IMAGE_MAP": str(image_map),
"FAKE_REPORT_DIR": str(reports),
"FAKE_DOCKER_LOG": str(docker_log),
"FAKE_TRIVY_LOG": str(trivy_log),
}
)
if trivy_fail_id is not None:
env["FAKE_TRIVY_FAIL_ID"] = trivy_fail_id
if curl_fail:
env["FAKE_CURL_FAIL"] = "1"
if curl_direct_fail:
env["FAKE_CURL_DIRECT_FAIL"] = "1"
if curl_api_fail:
env["FAKE_CURL_API_FAIL"] = "1"
if extra_env:
env.update(extra_env)
result = subprocess.run(
["sh", str(wrapper_copy), *images],
text=True,
capture_output=True,
env=env,
cwd=str(root) if run_in_workdir else None,
check=False,
)
if repeat_without_output:
self.assertEqual(result.returncode, 0, result.stderr)
env["FAKE_TRIVY_NO_OUTPUT_ID"] = CLEAN_ID
result = subprocess.run(["sh", str(wrapper_copy), *images], text=True, capture_output=True,
env=env, cwd=str(root) if run_in_workdir else None, check=False)
docker_calls = []
if docker_log.exists():
docker_calls = [
json.loads(line)
for line in docker_log.read_text(encoding="utf-8").splitlines()
if line.strip()
]
trivy_calls = []
if trivy_log.exists():
trivy_calls = [
json.loads(line)
for line in trivy_log.read_text(encoding="utf-8").splitlines()
if line.strip()
]
receipt = None
runs = sorted(evidence.glob("run.*"), key=lambda path: path.stat().st_mtime_ns)
if runs:
evidence = runs[-1]
receipt_path = evidence / "receipt.json"
if receipt_path.exists():
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
return result, root, evidence, receipt, docker_calls, trivy_calls
@staticmethod
def scan_calls(trivy_calls):
return [call for call in trivy_calls if call["args"][:1] == ["image"] and "--output" in call["args"]]
@staticmethod
def target_of(call):
return call["args"][-1]
def test_unfixed_high_blocks_overall_but_both_images_scanned(self):
result, _, evidence, receipt, docker_calls, trivy_calls = self.run_scan()
self.assertEqual(result.returncode, 1, result.stderr)
scans = self.scan_calls(trivy_calls)
self.assertEqual({self.target_of(call) for call in scans}, {VULN_ID, CLEAN_ID})
self.assertEqual(
{call[-1] for call in docker_calls if call[:2] == ["image", "inspect"]},
{VULN_TAG, CLEAN_TAG},
)
self.assertIsNotNone(receipt)
self.assertEqual(receipt["overall"]["exit_code"], 1)
self.assertEqual(receipt["overall"]["status"], "blocked")
self.assertEqual(len(receipt["images"]), 2)
self.assertEqual({entry["tag"] for entry in receipt["images"]}, {VULN_TAG, CLEAN_TAG})
self.assertIn(VULN_ID, {entry["image_id"] for entry in receipt["images"]})
by_tag = {entry["tag"]: entry for entry in receipt["images"]}
for entry in receipt["images"]:
self.assertTrue(Path(entry["raw"]).is_file(), entry["raw"])
self.assertTrue(Path(entry["decision"]).is_file(), entry["decision"])
self.assertTrue(Path(entry["scan_log"]).is_file(), entry["scan_log"])
self.assertEqual(by_tag[VULN_TAG]["status"], "blocked")
self.assertEqual(by_tag[CLEAN_TAG]["status"], "pass")
vuln_raw = json.loads(Path(by_tag[VULN_TAG]["raw"]).read_text(encoding="utf-8"))
clean_raw = json.loads(Path(by_tag[CLEAN_TAG]["raw"]).read_text(encoding="utf-8"))
self.assertEqual(vuln_raw["Metadata"]["ImageID"], VULN_ID)
self.assertEqual(clean_raw["Metadata"]["ImageID"], CLEAN_ID)
vuln_decision = json.loads(Path(by_tag[VULN_TAG]["decision"]).read_text(encoding="utf-8"))
clean_decision = json.loads(Path(by_tag[CLEAN_TAG]["decision"]).read_text(encoding="utf-8"))
self.assertEqual(vuln_decision["status"], "blocked")
self.assertEqual(vuln_decision["blocking"][0]["vulnerability_id"], "CVE-2024-0001")
self.assertEqual(clean_decision["status"], "pass")
def test_pinned_flags_survive_hidden_trivy_overrides(self):
hostile = {
"TRIVY_SEVERITY": "LOW",
"TRIVY_IGNORE_UNFIXED": "true",
"TRIVY_CONFIG": "/dev/null",
"TRIVY_SKIP_FILES": "usr/bin/infocmp",
"TRIVY_VEX": "vex.json",
"TRIVY_IGNORE_POLICY": "policy.rego",
}
result, _, _, _, _, trivy_calls = self.run_scan(
extra_env=hostile,
workdir_files={"trivy.yaml": "severity: LOW\nignore-unfixed: true\n"},
run_in_workdir=True,
)
self.assertEqual(result.returncode, 1, result.stderr)
for call in trivy_calls:
args = call["args"]
self.assertIn("--config", args)
self.assertEqual(args[args.index("--config") + 1], "/dev/null")
scans = self.scan_calls(trivy_calls)
self.assertTrue(scans)
for call in scans:
args = call["args"]
self.assertEqual(call["trivy_env"], [])
self.assertIn("--image-src", args)
self.assertEqual(args[args.index("--image-src") + 1], "docker")
self.assertEqual(args[args.index("--config") + 1], "/dev/null")
self.assertEqual(args[args.index("--ignorefile") + 1], "/dev/null")
self.assertIn("--ignore-unfixed=false", args)
self.assertIn("--list-all-pkgs=true", args)
self.assertIn("--skip-db-update", args)
self.assertIn("--skip-version-check", args)
self.assertIn("--no-progress", args)
self.assertEqual(args[args.index("--exit-code") + 1], "0")
self.assertEqual(args[args.index("--scanners") + 1], "vuln")
self.assertEqual(args[args.index("--format") + 1], "json")
self.assertEqual(
args[args.index("--severity") + 1],
"UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL",
)
def test_scan_error_on_first_image_does_not_stop_the_second(self):
result, _, _, receipt, _, trivy_calls = self.run_scan(trivy_fail_id=VULN_ID)
self.assertEqual(result.returncode, 2, result.stderr)
scans = self.scan_calls(trivy_calls)
self.assertEqual({self.target_of(call) for call in scans}, {VULN_ID, CLEAN_ID})
self.assertIsNotNone(receipt)
self.assertEqual(receipt["overall"]["exit_code"], 2)
self.assertEqual(receipt["overall"]["status"], "error")
by_tag = {entry["tag"]: entry for entry in receipt["images"]}
self.assertEqual(by_tag[VULN_TAG]["status"], "error")
self.assertEqual(by_tag[CLEAN_TAG]["status"], "pass")
clean_decision = json.loads(
Path(by_tag[CLEAN_TAG]["decision"]).read_text(encoding="utf-8")
)
self.assertEqual(clean_decision["status"], "pass")
vuln_decision = json.loads(
Path(by_tag[VULN_TAG]["decision"]).read_text(encoding="utf-8")
)
self.assertEqual(vuln_decision["status"], "error")
def test_scanner_version_failure_is_a_bootstrap_error(self):
result, _, _, receipt, _, _ = self.run_scan(extra_env={"FAKE_TRIVY_VERSION_FAIL": "1"})
self.assertEqual(result.returncode, 2, result.stderr)
self.assertEqual(receipt["overall"]["status"], "error")
def test_database_download_uses_official_priority_and_fallback(self):
result, _, _, _, _, trivy_calls = self.run_scan(images=(CLEAN_TAG,))
self.assertEqual(result.returncode, 0, result.stderr)
downloads = [call["args"] for call in trivy_calls if "--download-db-only" in call["args"]]
self.assertEqual(len(downloads), 1)
args = downloads[0]
repositories = [args[index + 1] for index, item in enumerate(args) if item == "--db-repository"]
self.assertEqual(
repositories,
["ghcr.io/aquasecurity/trivy-db:2", "mirror.gcr.io/aquasec/trivy-db:2"],
)
def test_rerun_cannot_reuse_an_old_successful_raw_report(self):
result, _, evidence, receipt, _, _ = self.run_scan(images=(CLEAN_TAG,), repeat_without_output=True)
self.assertEqual(result.returncode, 2, result.stderr)
self.assertEqual(receipt["overall"]["status"], "error")
self.assertEqual(receipt["images"][0]["status"], "error")
def test_bootstrap_failure_writes_error_receipt_for_each_image(self):
result, _, _, receipt, docker_calls, trivy_calls = self.run_scan(curl_fail=True)
self.assertEqual(result.returncode, 2, result.stderr)
self.assertIsNotNone(receipt)
self.assertEqual(receipt["overall"]["status"], "error")
self.assertEqual(receipt["overall"]["exit_code"], 2)
self.assertEqual(len(receipt["images"]), 2)
self.assertTrue(all(entry["status"] == "error" for entry in receipt["images"]))
self.assertEqual({entry["tag"] for entry in receipt["images"]}, {VULN_TAG, CLEAN_TAG})
self.assertEqual(docker_calls, [])
self.assertEqual(trivy_calls, [])
def test_pinned_asset_api_fallback_downloads_and_verifies_same_archive(self):
result, _, evidence, receipt, _, trivy_calls = self.run_scan(
images=(CLEAN_TAG,), curl_direct_fail=True
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(receipt["overall"]["status"], "pass")
self.assertIn("fixture download failure", (evidence / "scanner-download.log").read_text())
self.assertIn("GitHub release asset API", (evidence / "scanner-api-download.log").read_text())
self.assertTrue(any("--download-db-only" in call["args"] for call in trivy_calls))
scanner = json.loads((evidence / "scanner.json").read_text(encoding="utf-8"))
self.assertEqual(scanner["expected_version"], "0.70.0")
def test_pinned_asset_api_fallback_failure_fails_closed_with_both_logs(self):
result, _, evidence, receipt, _, trivy_calls = self.run_scan(
images=(CLEAN_TAG,), curl_direct_fail=True, curl_api_fail=True
)
self.assertEqual(result.returncode, 2)
self.assertEqual(receipt["overall"]["status"], "error")
self.assertIn("fixture download failure", (evidence / "scanner-download.log").read_text())
self.assertIn("fixture download failure", (evidence / "scanner-api-download.log").read_text())
self.assertEqual(trivy_calls, [])
def test_pinned_asset_api_fallback_checksum_mismatch_never_executes_trivy(self):
result, _, evidence, receipt, _, trivy_calls = self.run_scan(
images=(CLEAN_TAG,), curl_direct_fail=True, api_checksum_mismatch=True
)
self.assertEqual(result.returncode, 2)
self.assertEqual(receipt["overall"]["status"], "error")
self.assertIn("checksum mismatch", result.stderr)
self.assertEqual(trivy_calls, [])
def test_preloaded_pinned_archive_runs_without_network(self):
result, root, evidence, receipt, _, trivy_calls = self.run_scan(
images=(CLEAN_TAG,), cache_archive="valid", curl_fail=True
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(receipt["overall"]["status"], "pass")
self.assertEqual(
(evidence / "scanner-cache-source.log").read_text(),
"archive_source=runner-tool-cache\n",
)
self.assertFalse((evidence / "scanner-download.log").exists())
self.assertFalse((evidence / "scanner-api-download.log").exists())
cached = root / "runner-tool-cache" / "trivy" / "0.70.0" / "trivy_0.70.0_Linux-64bit.tar.gz"
self.assertEqual(cached.read_bytes(), (root / "trivy-fixture.tar.gz").read_bytes())
self.assertTrue(any("--download-db-only" in call["args"] for call in trivy_calls))
def test_preloaded_archive_hash_mismatch_fails_before_network_or_execution(self):
result, _, evidence, receipt, _, trivy_calls = self.run_scan(
images=(CLEAN_TAG,), cache_archive="bad", curl_fail=True
)
self.assertEqual(result.returncode, 2)
self.assertEqual(receipt["overall"]["status"], "error")
self.assertIn("checksum mismatch", result.stderr)
self.assertEqual(
(evidence / "scanner-cache-source.log").read_text(),
"archive_source=runner-tool-cache\n",
)
self.assertFalse((evidence / "scanner-download.log").exists())
self.assertFalse((evidence / "scanner-api-download.log").exists())
self.assertEqual(trivy_calls, [])
def test_unknown_image_is_error_and_is_never_scanned(self):
missing = "crank/missing:ci"
result, _, _, receipt, _, trivy_calls = self.run_scan(images=(missing, CLEAN_TAG))
self.assertEqual(result.returncode, 2, result.stderr)
by_tag = {entry["tag"]: entry for entry in receipt["images"]}
self.assertEqual(by_tag[missing]["status"], "error")
self.assertEqual(by_tag[CLEAN_TAG]["status"], "pass")
self.assertEqual([self.target_of(call) for call in self.scan_calls(trivy_calls)], [CLEAN_ID])
decision = json.loads(Path(by_tag[missing]["decision"]).read_text())
self.assertEqual(decision["status"], "error")
def test_download_error_log_is_retained(self):
result, _, evidence, receipt, _, _ = self.run_scan(curl_fail=True)
self.assertEqual(result.returncode, 2, result.stderr)
self.assertEqual(receipt["overall"]["status"], "error")
self.assertIn("fixture download failure", (evidence / "scanner-download.log").read_text())
self.assertIn("fixture download failure", (evidence / "scanner-api-download.log").read_text())
def test_evidence_kept_even_when_gate_is_red(self):
_, _, evidence, receipt, _, _ = self.run_scan()
self.assertTrue((evidence / "receipt.json").is_file())
self.assertTrue((evidence / "scanner.json").is_file())
self.assertTrue((evidence / "db" / "metadata.json").is_file())
scanner = json.loads((evidence / "scanner.json").read_text(encoding="utf-8"))
self.assertEqual(scanner["expected_version"], "0.70.0")
self.assertEqual(scanner["policy"]["ignore_unfixed"], False)
self.assertIsNotNone(receipt)
if __name__ == "__main__":
unittest.main()