Files
crank/tests/unit/test_scan_images.py
T
bsodfather f685616ae8
CI / Rust Checks (pull_request) Successful in 23m1s
CI / UI Checks (pull_request) Successful in 6s
CI / Community Image Smoke (pull_request) Successful in 7m28s
CI / Frontend E2E (pull_request) Successful in 9m33s
CI / Deploy (pull_request) Skipped
fix(security): minimize runtime images and enforce complete vulnerability scans
2026-10-10 01:41:38 +03:00

463 lines
18 KiB
Python

import hashlib
import io
import json
import os
import re
import shutil
import subprocess
import tarfile
import tempfile
import unittest
from datetime import datetime, timedelta, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
WRAPPER = ROOT / "scripts" / "scan-images.sh"
CHECKER = ROOT / "scripts" / "check-image-scan.py"
VULN_ID = "sha256:" + "1" * 64
CLEAN_ID = "sha256:" + "2" * 64
VULN_TAG = "crank/vuln:ci"
CLEAN_TAG = "crank/clean:ci"
FAKE_CURL = """#!/usr/bin/env python3
import os
import sys
args = sys.argv[1:]
if os.environ.get("FAKE_CURL_FAIL") == "1":
sys.stderr.write("curl: (7) failed to connect\\n")
sys.exit(7)
output = None
for index, value in enumerate(args):
if value in ("--output", "-o"):
output = args[index + 1]
if output is None:
sys.stderr.write("curl: missing --output\\n")
sys.exit(2)
with open(os.environ["FAKE_CURL_ARCHIVE"], "rb") as source:
payload = source.read()
with open(output, "wb") as target:
target.write(payload)
sys.exit(0)
"""
FAKE_DOCKER = """#!/usr/bin/env python3
import json
import os
import sys
args = sys.argv[1:]
with open(os.environ["FAKE_DOCKER_LOG"], "a", encoding="utf-8") as stream:
stream.write(json.dumps(args) + "\\n")
if args[:2] == ["image", "inspect"]:
with open(os.environ["FAKE_IMAGE_MAP"], encoding="utf-8") as source:
mapping = json.load(source)
entry = mapping.get(args[-1])
if entry is None:
sys.stderr.write("docker: no such image " + args[-1] + "\\n")
sys.exit(1)
sys.stdout.write(entry["id"] + "\\n")
sys.exit(0)
sys.stderr.write("docker: unexpected command\\n")
sys.exit(2)
"""
# Fake Trivy shipped inside the stub scanner archive. It records every
# invocation and any inherited TRIVY_* variable so the test can prove the
# wrapper both pinned the flags and scrubbed the environment.
FAKE_TRIVY = """#!/usr/bin/env python3
import json
import os
import sys
from datetime import datetime, timedelta, timezone
args = sys.argv[1:]
log = os.environ.get("FAKE_TRIVY_LOG")
if log:
record = {
"args": args,
"trivy_env": sorted(name for name in os.environ if name.startswith("TRIVY_")),
}
with open(log, "a", encoding="utf-8") as stream:
stream.write(json.dumps(record) + "\\n")
if "--version" in args:
if os.environ.get("FAKE_TRIVY_VERSION_FAIL"):
sys.exit(7)
sys.stdout.write("Version: 0.70.0\\n")
sys.exit(0)
if "--download-db-only" in args:
cache = args[args.index("--cache-dir") + 1]
db_dir = os.path.join(cache, "db")
os.makedirs(db_dir, exist_ok=True)
with open(os.path.join(db_dir, "trivy.db"), "w", encoding="utf-8") as stream:
stream.write("fixture-db\\n")
now = datetime.now(timezone.utc)
metadata = {
"Version": 2,
"UpdatedAt": now.isoformat().replace("+00:00", "Z"),
"NextUpdate": (now + timedelta(hours=24)).isoformat().replace("+00:00", "Z"),
}
with open(os.path.join(db_dir, "metadata.json"), "w", encoding="utf-8") as stream:
json.dump(metadata, stream)
sys.exit(0)
target = args[-1]
if os.environ.get("FAKE_TRIVY_NO_OUTPUT_ID") == target:
sys.exit(0)
if os.environ.get("FAKE_TRIVY_FAIL_ID") == target:
sys.stderr.write("trivy: fixture scan failure for " + target + "\\n")
sys.exit(1)
output = args[args.index("--output") + 1]
source = os.path.join(os.environ["FAKE_REPORT_DIR"], target.replace(":", "_") + ".json")
if not os.path.exists(source):
sys.stderr.write("trivy: no report fixture for " + target + "\\n")
sys.exit(1)
with open(source, encoding="utf-8") as stream:
report = json.load(stream)
report["Metadata"]["ImageID"] = target
with open(output, "w", encoding="utf-8") as stream:
json.dump(report, stream)
sys.exit(0)
"""
def build_archive(script: str) -> bytes:
buffer = io.BytesIO()
with tarfile.open(fileobj=buffer, mode="w:gz") as archive:
data = script.encode("utf-8")
info = tarfile.TarInfo("trivy")
info.size = len(data)
info.mode = 0o755
info.type = tarfile.REGTYPE
archive.addfile(info, io.BytesIO(data))
return buffer.getvalue()
def alpine_report(image_id: str, vulnerabilities=None) -> dict:
result = {
"Target": "fixture:latest (alpine 3.24)",
"Class": "os-pkgs",
"Type": "alpine",
"Packages": [
{"ID": "musl@1.2.5-r0", "Name": "musl", "Version": "1.2.5-r0"}
],
}
if vulnerabilities is not None:
result["Vulnerabilities"] = vulnerabilities
return {
"SchemaVersion": 2,
"ArtifactName": "fixture:latest",
"ArtifactType": "container_image",
"Metadata": {
"ImageID": image_id,
"OS": {"Family": "alpine", "Name": "3.24"},
},
"Results": [result],
}
def unfixed_high() -> dict:
return {
"VulnerabilityID": "CVE-2024-0001",
"PkgName": "musl",
"InstalledVersion": "1.2.5-r0",
"Severity": "HIGH",
"Status": "affected",
"Title": "fixture unfixed high",
}
class ScanImagesWrapperTests(unittest.TestCase):
def run_scan(
self,
*,
images=(VULN_TAG, CLEAN_TAG),
trivy_fail_id=None,
curl_fail=False,
extra_env=None,
workdir_files=None,
run_in_workdir=False,
repeat_without_output=False,
):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
root = Path(temporary.name)
bin_dir = root / "bin"
bin_dir.mkdir()
for name, contents in (("curl", FAKE_CURL), ("docker", FAKE_DOCKER)):
path = bin_dir / name
path.write_text(contents, encoding="utf-8")
path.chmod(0o755)
wrappers = root / "wrappers"
wrappers.mkdir()
fixture_archive = build_archive(FAKE_TRIVY)
production = WRAPPER.read_text(encoding="utf-8")
fixture_hash = hashlib.sha256(fixture_archive).hexdigest()
# Replace only the pinned archive hash in the test copy; the production
# wrapper is never modified and there is no bypass environment variable.
patched, replacements = re.subn(
r'expected_sha256="[0-9a-f]{64}"',
f'expected_sha256="{fixture_hash}"',
production,
count=1,
)
self.assertEqual(replacements, 1)
self.assertIn('expected_sha256="8b4376d5d6befe5c24d503f10ff136d9e0c49f9127a4279fd110b727929a5aa9"', production)
wrapper_copy = wrappers / "scan-images.sh"
wrapper_copy.write_text(patched, encoding="utf-8")
wrapper_copy.chmod(0o755)
shutil.copy2(CHECKER, wrappers / "check-image-scan.py")
archive_path = root / "trivy-fixture.tar.gz"
archive_path.write_bytes(fixture_archive)
reports = root / "reports"
reports.mkdir()
(reports / (VULN_ID.replace(":", "_") + ".json")).write_text(
json.dumps(alpine_report(VULN_ID, [unfixed_high()])), encoding="utf-8"
)
(reports / (CLEAN_ID.replace(":", "_") + ".json")).write_text(
json.dumps(alpine_report(CLEAN_ID)), encoding="utf-8"
)
image_map = root / "images.json"
image_map.write_text(
json.dumps(
{
VULN_TAG: {"id": VULN_ID},
CLEAN_TAG: {"id": CLEAN_ID},
}
),
encoding="utf-8",
)
evidence = root / "evidence"
docker_log = root / "docker.jsonl"
trivy_log = root / "trivy.jsonl"
for name, contents in (workdir_files or {}).items():
(root / name).write_text(contents, encoding="utf-8")
env = os.environ.copy()
env.update(
{
"PATH": f"{bin_dir}:{env['PATH']}",
"CRANK_IMAGE_SCAN_OUTPUT_DIR": str(evidence),
"FAKE_CURL_ARCHIVE": str(archive_path),
"FAKE_IMAGE_MAP": str(image_map),
"FAKE_REPORT_DIR": str(reports),
"FAKE_DOCKER_LOG": str(docker_log),
"FAKE_TRIVY_LOG": str(trivy_log),
}
)
if trivy_fail_id is not None:
env["FAKE_TRIVY_FAIL_ID"] = trivy_fail_id
if curl_fail:
env["FAKE_CURL_FAIL"] = "1"
if extra_env:
env.update(extra_env)
result = subprocess.run(
["sh", str(wrapper_copy), *images],
text=True,
capture_output=True,
env=env,
cwd=str(root) if run_in_workdir else None,
check=False,
)
if repeat_without_output:
self.assertEqual(result.returncode, 0, result.stderr)
env["FAKE_TRIVY_NO_OUTPUT_ID"] = CLEAN_ID
result = subprocess.run(["sh", str(wrapper_copy), *images], text=True, capture_output=True,
env=env, cwd=str(root) if run_in_workdir else None, check=False)
docker_calls = []
if docker_log.exists():
docker_calls = [
json.loads(line)
for line in docker_log.read_text(encoding="utf-8").splitlines()
if line.strip()
]
trivy_calls = []
if trivy_log.exists():
trivy_calls = [
json.loads(line)
for line in trivy_log.read_text(encoding="utf-8").splitlines()
if line.strip()
]
receipt = None
runs = sorted(evidence.glob("run.*"), key=lambda path: path.stat().st_mtime_ns)
if runs:
evidence = runs[-1]
receipt_path = evidence / "receipt.json"
if receipt_path.exists():
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
return result, root, evidence, receipt, docker_calls, trivy_calls
@staticmethod
def scan_calls(trivy_calls):
return [call for call in trivy_calls if call["args"][:1] == ["image"] and "--output" in call["args"]]
@staticmethod
def target_of(call):
return call["args"][-1]
def test_unfixed_high_blocks_overall_but_both_images_scanned(self):
result, _, evidence, receipt, docker_calls, trivy_calls = self.run_scan()
self.assertEqual(result.returncode, 1, result.stderr)
scans = self.scan_calls(trivy_calls)
self.assertEqual({self.target_of(call) for call in scans}, {VULN_ID, CLEAN_ID})
self.assertEqual(
{call[-1] for call in docker_calls if call[:2] == ["image", "inspect"]},
{VULN_TAG, CLEAN_TAG},
)
self.assertIsNotNone(receipt)
self.assertEqual(receipt["overall"]["exit_code"], 1)
self.assertEqual(receipt["overall"]["status"], "blocked")
self.assertEqual(len(receipt["images"]), 2)
self.assertEqual({entry["tag"] for entry in receipt["images"]}, {VULN_TAG, CLEAN_TAG})
self.assertIn(VULN_ID, {entry["image_id"] for entry in receipt["images"]})
by_tag = {entry["tag"]: entry for entry in receipt["images"]}
for entry in receipt["images"]:
self.assertTrue(Path(entry["raw"]).is_file(), entry["raw"])
self.assertTrue(Path(entry["decision"]).is_file(), entry["decision"])
self.assertTrue(Path(entry["scan_log"]).is_file(), entry["scan_log"])
self.assertEqual(by_tag[VULN_TAG]["status"], "blocked")
self.assertEqual(by_tag[CLEAN_TAG]["status"], "pass")
vuln_raw = json.loads(Path(by_tag[VULN_TAG]["raw"]).read_text(encoding="utf-8"))
clean_raw = json.loads(Path(by_tag[CLEAN_TAG]["raw"]).read_text(encoding="utf-8"))
self.assertEqual(vuln_raw["Metadata"]["ImageID"], VULN_ID)
self.assertEqual(clean_raw["Metadata"]["ImageID"], CLEAN_ID)
vuln_decision = json.loads(Path(by_tag[VULN_TAG]["decision"]).read_text(encoding="utf-8"))
clean_decision = json.loads(Path(by_tag[CLEAN_TAG]["decision"]).read_text(encoding="utf-8"))
self.assertEqual(vuln_decision["status"], "blocked")
self.assertEqual(vuln_decision["blocking"][0]["vulnerability_id"], "CVE-2024-0001")
self.assertEqual(clean_decision["status"], "pass")
def test_pinned_flags_survive_hidden_trivy_overrides(self):
hostile = {
"TRIVY_SEVERITY": "LOW",
"TRIVY_IGNORE_UNFIXED": "true",
"TRIVY_CONFIG": "/dev/null",
"TRIVY_SKIP_FILES": "usr/bin/infocmp",
"TRIVY_VEX": "vex.json",
"TRIVY_IGNORE_POLICY": "policy.rego",
}
result, _, _, _, _, trivy_calls = self.run_scan(
extra_env=hostile,
workdir_files={"trivy.yaml": "severity: LOW\nignore-unfixed: true\n"},
run_in_workdir=True,
)
self.assertEqual(result.returncode, 1, result.stderr)
for call in trivy_calls:
args = call["args"]
self.assertIn("--config", args)
self.assertEqual(args[args.index("--config") + 1], "/dev/null")
scans = self.scan_calls(trivy_calls)
self.assertTrue(scans)
for call in scans:
args = call["args"]
self.assertEqual(call["trivy_env"], [])
self.assertIn("--image-src", args)
self.assertEqual(args[args.index("--image-src") + 1], "docker")
self.assertEqual(args[args.index("--config") + 1], "/dev/null")
self.assertEqual(args[args.index("--ignorefile") + 1], "/dev/null")
self.assertIn("--ignore-unfixed=false", args)
self.assertIn("--list-all-pkgs=true", args)
self.assertIn("--skip-db-update", args)
self.assertIn("--skip-version-check", args)
self.assertIn("--no-progress", args)
self.assertEqual(args[args.index("--exit-code") + 1], "0")
self.assertEqual(args[args.index("--scanners") + 1], "vuln")
self.assertEqual(args[args.index("--format") + 1], "json")
self.assertEqual(
args[args.index("--severity") + 1],
"UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL",
)
def test_scan_error_on_first_image_does_not_stop_the_second(self):
result, _, _, receipt, _, trivy_calls = self.run_scan(trivy_fail_id=VULN_ID)
self.assertEqual(result.returncode, 2, result.stderr)
scans = self.scan_calls(trivy_calls)
self.assertEqual({self.target_of(call) for call in scans}, {VULN_ID, CLEAN_ID})
self.assertIsNotNone(receipt)
self.assertEqual(receipt["overall"]["exit_code"], 2)
self.assertEqual(receipt["overall"]["status"], "error")
by_tag = {entry["tag"]: entry for entry in receipt["images"]}
self.assertEqual(by_tag[VULN_TAG]["status"], "error")
self.assertEqual(by_tag[CLEAN_TAG]["status"], "pass")
clean_decision = json.loads(
Path(by_tag[CLEAN_TAG]["decision"]).read_text(encoding="utf-8")
)
self.assertEqual(clean_decision["status"], "pass")
vuln_decision = json.loads(
Path(by_tag[VULN_TAG]["decision"]).read_text(encoding="utf-8")
)
self.assertEqual(vuln_decision["status"], "error")
def test_scanner_version_failure_is_a_bootstrap_error(self):
result, _, _, receipt, _, _ = self.run_scan(extra_env={"FAKE_TRIVY_VERSION_FAIL": "1"})
self.assertEqual(result.returncode, 2, result.stderr)
self.assertEqual(receipt["overall"]["status"], "error")
def test_rerun_cannot_reuse_an_old_successful_raw_report(self):
result, _, evidence, receipt, _, _ = self.run_scan(images=(CLEAN_TAG,), repeat_without_output=True)
self.assertEqual(result.returncode, 2, result.stderr)
self.assertEqual(receipt["overall"]["status"], "error")
self.assertEqual(receipt["images"][0]["status"], "error")
def test_bootstrap_failure_writes_error_receipt_for_each_image(self):
result, _, _, receipt, docker_calls, trivy_calls = self.run_scan(curl_fail=True)
self.assertEqual(result.returncode, 2, result.stderr)
self.assertIsNotNone(receipt)
self.assertEqual(receipt["overall"]["status"], "error")
self.assertEqual(receipt["overall"]["exit_code"], 2)
self.assertEqual(len(receipt["images"]), 2)
self.assertTrue(all(entry["status"] == "error" for entry in receipt["images"]))
self.assertEqual({entry["tag"] for entry in receipt["images"]}, {VULN_TAG, CLEAN_TAG})
self.assertEqual(docker_calls, [])
self.assertEqual(trivy_calls, [])
def test_unknown_image_is_error_and_is_never_scanned(self):
missing = "crank/missing:ci"
result, _, _, receipt, _, trivy_calls = self.run_scan(images=(missing, CLEAN_TAG))
self.assertEqual(result.returncode, 2, result.stderr)
by_tag = {entry["tag"]: entry for entry in receipt["images"]}
self.assertEqual(by_tag[missing]["status"], "error")
self.assertEqual(by_tag[CLEAN_TAG]["status"], "pass")
self.assertEqual([self.target_of(call) for call in self.scan_calls(trivy_calls)], [CLEAN_ID])
decision = json.loads(Path(by_tag[missing]["decision"]).read_text())
self.assertEqual(decision["status"], "error")
def test_download_error_log_is_retained(self):
result, _, evidence, receipt, _, _ = self.run_scan(curl_fail=True)
self.assertEqual(result.returncode, 2, result.stderr)
self.assertEqual(receipt["overall"]["status"], "error")
self.assertIn("failed to connect", (evidence / "scanner-download.log").read_text())
def test_evidence_kept_even_when_gate_is_red(self):
_, _, evidence, receipt, _, _ = self.run_scan()
self.assertTrue((evidence / "receipt.json").is_file())
self.assertTrue((evidence / "scanner.json").is_file())
self.assertTrue((evidence / "db" / "metadata.json").is_file())
scanner = json.loads((evidence / "scanner.json").read_text(encoding="utf-8"))
self.assertEqual(scanner["expected_version"], "0.70.0")
self.assertEqual(scanner["policy"]["ignore_unfixed"], False)
self.assertIsNotNone(receipt)
if __name__ == "__main__":
unittest.main()