463 lines
18 KiB
Python
463 lines
18 KiB
Python
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
import unittest
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
WRAPPER = ROOT / "scripts" / "scan-images.sh"
|
|
CHECKER = ROOT / "scripts" / "check-image-scan.py"
|
|
|
|
VULN_ID = "sha256:" + "1" * 64
|
|
CLEAN_ID = "sha256:" + "2" * 64
|
|
|
|
VULN_TAG = "crank/vuln:ci"
|
|
CLEAN_TAG = "crank/clean:ci"
|
|
|
|
FAKE_CURL = """#!/usr/bin/env python3
|
|
import os
|
|
import sys
|
|
|
|
args = sys.argv[1:]
|
|
if os.environ.get("FAKE_CURL_FAIL") == "1":
|
|
sys.stderr.write("curl: (7) failed to connect\\n")
|
|
sys.exit(7)
|
|
output = None
|
|
for index, value in enumerate(args):
|
|
if value in ("--output", "-o"):
|
|
output = args[index + 1]
|
|
if output is None:
|
|
sys.stderr.write("curl: missing --output\\n")
|
|
sys.exit(2)
|
|
with open(os.environ["FAKE_CURL_ARCHIVE"], "rb") as source:
|
|
payload = source.read()
|
|
with open(output, "wb") as target:
|
|
target.write(payload)
|
|
sys.exit(0)
|
|
"""
|
|
|
|
FAKE_DOCKER = """#!/usr/bin/env python3
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
args = sys.argv[1:]
|
|
with open(os.environ["FAKE_DOCKER_LOG"], "a", encoding="utf-8") as stream:
|
|
stream.write(json.dumps(args) + "\\n")
|
|
if args[:2] == ["image", "inspect"]:
|
|
with open(os.environ["FAKE_IMAGE_MAP"], encoding="utf-8") as source:
|
|
mapping = json.load(source)
|
|
entry = mapping.get(args[-1])
|
|
if entry is None:
|
|
sys.stderr.write("docker: no such image " + args[-1] + "\\n")
|
|
sys.exit(1)
|
|
sys.stdout.write(entry["id"] + "\\n")
|
|
sys.exit(0)
|
|
sys.stderr.write("docker: unexpected command\\n")
|
|
sys.exit(2)
|
|
"""
|
|
|
|
# Fake Trivy shipped inside the stub scanner archive. It records every
|
|
# invocation and any inherited TRIVY_* variable so the test can prove the
|
|
# wrapper both pinned the flags and scrubbed the environment.
|
|
FAKE_TRIVY = """#!/usr/bin/env python3
|
|
import json
|
|
import os
|
|
import sys
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
args = sys.argv[1:]
|
|
log = os.environ.get("FAKE_TRIVY_LOG")
|
|
if log:
|
|
record = {
|
|
"args": args,
|
|
"trivy_env": sorted(name for name in os.environ if name.startswith("TRIVY_")),
|
|
}
|
|
with open(log, "a", encoding="utf-8") as stream:
|
|
stream.write(json.dumps(record) + "\\n")
|
|
if "--version" in args:
|
|
if os.environ.get("FAKE_TRIVY_VERSION_FAIL"):
|
|
sys.exit(7)
|
|
sys.stdout.write("Version: 0.70.0\\n")
|
|
sys.exit(0)
|
|
if "--download-db-only" in args:
|
|
cache = args[args.index("--cache-dir") + 1]
|
|
db_dir = os.path.join(cache, "db")
|
|
os.makedirs(db_dir, exist_ok=True)
|
|
with open(os.path.join(db_dir, "trivy.db"), "w", encoding="utf-8") as stream:
|
|
stream.write("fixture-db\\n")
|
|
now = datetime.now(timezone.utc)
|
|
metadata = {
|
|
"Version": 2,
|
|
"UpdatedAt": now.isoformat().replace("+00:00", "Z"),
|
|
"NextUpdate": (now + timedelta(hours=24)).isoformat().replace("+00:00", "Z"),
|
|
}
|
|
with open(os.path.join(db_dir, "metadata.json"), "w", encoding="utf-8") as stream:
|
|
json.dump(metadata, stream)
|
|
sys.exit(0)
|
|
target = args[-1]
|
|
if os.environ.get("FAKE_TRIVY_NO_OUTPUT_ID") == target:
|
|
sys.exit(0)
|
|
if os.environ.get("FAKE_TRIVY_FAIL_ID") == target:
|
|
sys.stderr.write("trivy: fixture scan failure for " + target + "\\n")
|
|
sys.exit(1)
|
|
output = args[args.index("--output") + 1]
|
|
source = os.path.join(os.environ["FAKE_REPORT_DIR"], target.replace(":", "_") + ".json")
|
|
if not os.path.exists(source):
|
|
sys.stderr.write("trivy: no report fixture for " + target + "\\n")
|
|
sys.exit(1)
|
|
with open(source, encoding="utf-8") as stream:
|
|
report = json.load(stream)
|
|
report["Metadata"]["ImageID"] = target
|
|
with open(output, "w", encoding="utf-8") as stream:
|
|
json.dump(report, stream)
|
|
sys.exit(0)
|
|
"""
|
|
|
|
|
|
def build_archive(script: str) -> bytes:
|
|
buffer = io.BytesIO()
|
|
with tarfile.open(fileobj=buffer, mode="w:gz") as archive:
|
|
data = script.encode("utf-8")
|
|
info = tarfile.TarInfo("trivy")
|
|
info.size = len(data)
|
|
info.mode = 0o755
|
|
info.type = tarfile.REGTYPE
|
|
archive.addfile(info, io.BytesIO(data))
|
|
return buffer.getvalue()
|
|
|
|
|
|
def alpine_report(image_id: str, vulnerabilities=None) -> dict:
|
|
result = {
|
|
"Target": "fixture:latest (alpine 3.24)",
|
|
"Class": "os-pkgs",
|
|
"Type": "alpine",
|
|
"Packages": [
|
|
{"ID": "musl@1.2.5-r0", "Name": "musl", "Version": "1.2.5-r0"}
|
|
],
|
|
}
|
|
if vulnerabilities is not None:
|
|
result["Vulnerabilities"] = vulnerabilities
|
|
return {
|
|
"SchemaVersion": 2,
|
|
"ArtifactName": "fixture:latest",
|
|
"ArtifactType": "container_image",
|
|
"Metadata": {
|
|
"ImageID": image_id,
|
|
"OS": {"Family": "alpine", "Name": "3.24"},
|
|
},
|
|
"Results": [result],
|
|
}
|
|
|
|
|
|
def unfixed_high() -> dict:
|
|
return {
|
|
"VulnerabilityID": "CVE-2024-0001",
|
|
"PkgName": "musl",
|
|
"InstalledVersion": "1.2.5-r0",
|
|
"Severity": "HIGH",
|
|
"Status": "affected",
|
|
"Title": "fixture unfixed high",
|
|
}
|
|
|
|
|
|
class ScanImagesWrapperTests(unittest.TestCase):
|
|
def run_scan(
|
|
self,
|
|
*,
|
|
images=(VULN_TAG, CLEAN_TAG),
|
|
trivy_fail_id=None,
|
|
curl_fail=False,
|
|
extra_env=None,
|
|
workdir_files=None,
|
|
run_in_workdir=False,
|
|
repeat_without_output=False,
|
|
):
|
|
temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
root = Path(temporary.name)
|
|
|
|
bin_dir = root / "bin"
|
|
bin_dir.mkdir()
|
|
for name, contents in (("curl", FAKE_CURL), ("docker", FAKE_DOCKER)):
|
|
path = bin_dir / name
|
|
path.write_text(contents, encoding="utf-8")
|
|
path.chmod(0o755)
|
|
|
|
wrappers = root / "wrappers"
|
|
wrappers.mkdir()
|
|
fixture_archive = build_archive(FAKE_TRIVY)
|
|
production = WRAPPER.read_text(encoding="utf-8")
|
|
fixture_hash = hashlib.sha256(fixture_archive).hexdigest()
|
|
# Replace only the pinned archive hash in the test copy; the production
|
|
# wrapper is never modified and there is no bypass environment variable.
|
|
patched, replacements = re.subn(
|
|
r'expected_sha256="[0-9a-f]{64}"',
|
|
f'expected_sha256="{fixture_hash}"',
|
|
production,
|
|
count=1,
|
|
)
|
|
self.assertEqual(replacements, 1)
|
|
self.assertIn('expected_sha256="8b4376d5d6befe5c24d503f10ff136d9e0c49f9127a4279fd110b727929a5aa9"', production)
|
|
wrapper_copy = wrappers / "scan-images.sh"
|
|
wrapper_copy.write_text(patched, encoding="utf-8")
|
|
wrapper_copy.chmod(0o755)
|
|
shutil.copy2(CHECKER, wrappers / "check-image-scan.py")
|
|
|
|
archive_path = root / "trivy-fixture.tar.gz"
|
|
archive_path.write_bytes(fixture_archive)
|
|
|
|
reports = root / "reports"
|
|
reports.mkdir()
|
|
(reports / (VULN_ID.replace(":", "_") + ".json")).write_text(
|
|
json.dumps(alpine_report(VULN_ID, [unfixed_high()])), encoding="utf-8"
|
|
)
|
|
(reports / (CLEAN_ID.replace(":", "_") + ".json")).write_text(
|
|
json.dumps(alpine_report(CLEAN_ID)), encoding="utf-8"
|
|
)
|
|
|
|
image_map = root / "images.json"
|
|
image_map.write_text(
|
|
json.dumps(
|
|
{
|
|
VULN_TAG: {"id": VULN_ID},
|
|
CLEAN_TAG: {"id": CLEAN_ID},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
evidence = root / "evidence"
|
|
docker_log = root / "docker.jsonl"
|
|
trivy_log = root / "trivy.jsonl"
|
|
|
|
for name, contents in (workdir_files or {}).items():
|
|
(root / name).write_text(contents, encoding="utf-8")
|
|
|
|
env = os.environ.copy()
|
|
env.update(
|
|
{
|
|
"PATH": f"{bin_dir}:{env['PATH']}",
|
|
"CRANK_IMAGE_SCAN_OUTPUT_DIR": str(evidence),
|
|
"FAKE_CURL_ARCHIVE": str(archive_path),
|
|
"FAKE_IMAGE_MAP": str(image_map),
|
|
"FAKE_REPORT_DIR": str(reports),
|
|
"FAKE_DOCKER_LOG": str(docker_log),
|
|
"FAKE_TRIVY_LOG": str(trivy_log),
|
|
}
|
|
)
|
|
if trivy_fail_id is not None:
|
|
env["FAKE_TRIVY_FAIL_ID"] = trivy_fail_id
|
|
if curl_fail:
|
|
env["FAKE_CURL_FAIL"] = "1"
|
|
if extra_env:
|
|
env.update(extra_env)
|
|
|
|
result = subprocess.run(
|
|
["sh", str(wrapper_copy), *images],
|
|
text=True,
|
|
capture_output=True,
|
|
env=env,
|
|
cwd=str(root) if run_in_workdir else None,
|
|
check=False,
|
|
)
|
|
if repeat_without_output:
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
env["FAKE_TRIVY_NO_OUTPUT_ID"] = CLEAN_ID
|
|
result = subprocess.run(["sh", str(wrapper_copy), *images], text=True, capture_output=True,
|
|
env=env, cwd=str(root) if run_in_workdir else None, check=False)
|
|
|
|
docker_calls = []
|
|
if docker_log.exists():
|
|
docker_calls = [
|
|
json.loads(line)
|
|
for line in docker_log.read_text(encoding="utf-8").splitlines()
|
|
if line.strip()
|
|
]
|
|
trivy_calls = []
|
|
if trivy_log.exists():
|
|
trivy_calls = [
|
|
json.loads(line)
|
|
for line in trivy_log.read_text(encoding="utf-8").splitlines()
|
|
if line.strip()
|
|
]
|
|
receipt = None
|
|
runs = sorted(evidence.glob("run.*"), key=lambda path: path.stat().st_mtime_ns)
|
|
if runs:
|
|
evidence = runs[-1]
|
|
receipt_path = evidence / "receipt.json"
|
|
if receipt_path.exists():
|
|
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
|
|
return result, root, evidence, receipt, docker_calls, trivy_calls
|
|
|
|
@staticmethod
|
|
def scan_calls(trivy_calls):
|
|
return [call for call in trivy_calls if call["args"][:1] == ["image"] and "--output" in call["args"]]
|
|
|
|
@staticmethod
|
|
def target_of(call):
|
|
return call["args"][-1]
|
|
|
|
def test_unfixed_high_blocks_overall_but_both_images_scanned(self):
|
|
result, _, evidence, receipt, docker_calls, trivy_calls = self.run_scan()
|
|
self.assertEqual(result.returncode, 1, result.stderr)
|
|
|
|
scans = self.scan_calls(trivy_calls)
|
|
self.assertEqual({self.target_of(call) for call in scans}, {VULN_ID, CLEAN_ID})
|
|
self.assertEqual(
|
|
{call[-1] for call in docker_calls if call[:2] == ["image", "inspect"]},
|
|
{VULN_TAG, CLEAN_TAG},
|
|
)
|
|
|
|
self.assertIsNotNone(receipt)
|
|
self.assertEqual(receipt["overall"]["exit_code"], 1)
|
|
self.assertEqual(receipt["overall"]["status"], "blocked")
|
|
self.assertEqual(len(receipt["images"]), 2)
|
|
self.assertEqual({entry["tag"] for entry in receipt["images"]}, {VULN_TAG, CLEAN_TAG})
|
|
self.assertIn(VULN_ID, {entry["image_id"] for entry in receipt["images"]})
|
|
|
|
by_tag = {entry["tag"]: entry for entry in receipt["images"]}
|
|
for entry in receipt["images"]:
|
|
self.assertTrue(Path(entry["raw"]).is_file(), entry["raw"])
|
|
self.assertTrue(Path(entry["decision"]).is_file(), entry["decision"])
|
|
self.assertTrue(Path(entry["scan_log"]).is_file(), entry["scan_log"])
|
|
self.assertEqual(by_tag[VULN_TAG]["status"], "blocked")
|
|
self.assertEqual(by_tag[CLEAN_TAG]["status"], "pass")
|
|
|
|
vuln_raw = json.loads(Path(by_tag[VULN_TAG]["raw"]).read_text(encoding="utf-8"))
|
|
clean_raw = json.loads(Path(by_tag[CLEAN_TAG]["raw"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(vuln_raw["Metadata"]["ImageID"], VULN_ID)
|
|
self.assertEqual(clean_raw["Metadata"]["ImageID"], CLEAN_ID)
|
|
|
|
vuln_decision = json.loads(Path(by_tag[VULN_TAG]["decision"]).read_text(encoding="utf-8"))
|
|
clean_decision = json.loads(Path(by_tag[CLEAN_TAG]["decision"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(vuln_decision["status"], "blocked")
|
|
self.assertEqual(vuln_decision["blocking"][0]["vulnerability_id"], "CVE-2024-0001")
|
|
self.assertEqual(clean_decision["status"], "pass")
|
|
|
|
def test_pinned_flags_survive_hidden_trivy_overrides(self):
|
|
hostile = {
|
|
"TRIVY_SEVERITY": "LOW",
|
|
"TRIVY_IGNORE_UNFIXED": "true",
|
|
"TRIVY_CONFIG": "/dev/null",
|
|
"TRIVY_SKIP_FILES": "usr/bin/infocmp",
|
|
"TRIVY_VEX": "vex.json",
|
|
"TRIVY_IGNORE_POLICY": "policy.rego",
|
|
}
|
|
result, _, _, _, _, trivy_calls = self.run_scan(
|
|
extra_env=hostile,
|
|
workdir_files={"trivy.yaml": "severity: LOW\nignore-unfixed: true\n"},
|
|
run_in_workdir=True,
|
|
)
|
|
self.assertEqual(result.returncode, 1, result.stderr)
|
|
for call in trivy_calls:
|
|
args = call["args"]
|
|
self.assertIn("--config", args)
|
|
self.assertEqual(args[args.index("--config") + 1], "/dev/null")
|
|
|
|
scans = self.scan_calls(trivy_calls)
|
|
self.assertTrue(scans)
|
|
for call in scans:
|
|
args = call["args"]
|
|
self.assertEqual(call["trivy_env"], [])
|
|
self.assertIn("--image-src", args)
|
|
self.assertEqual(args[args.index("--image-src") + 1], "docker")
|
|
self.assertEqual(args[args.index("--config") + 1], "/dev/null")
|
|
self.assertEqual(args[args.index("--ignorefile") + 1], "/dev/null")
|
|
self.assertIn("--ignore-unfixed=false", args)
|
|
self.assertIn("--list-all-pkgs=true", args)
|
|
self.assertIn("--skip-db-update", args)
|
|
self.assertIn("--skip-version-check", args)
|
|
self.assertIn("--no-progress", args)
|
|
self.assertEqual(args[args.index("--exit-code") + 1], "0")
|
|
self.assertEqual(args[args.index("--scanners") + 1], "vuln")
|
|
self.assertEqual(args[args.index("--format") + 1], "json")
|
|
self.assertEqual(
|
|
args[args.index("--severity") + 1],
|
|
"UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL",
|
|
)
|
|
|
|
def test_scan_error_on_first_image_does_not_stop_the_second(self):
|
|
result, _, _, receipt, _, trivy_calls = self.run_scan(trivy_fail_id=VULN_ID)
|
|
self.assertEqual(result.returncode, 2, result.stderr)
|
|
|
|
scans = self.scan_calls(trivy_calls)
|
|
self.assertEqual({self.target_of(call) for call in scans}, {VULN_ID, CLEAN_ID})
|
|
|
|
self.assertIsNotNone(receipt)
|
|
self.assertEqual(receipt["overall"]["exit_code"], 2)
|
|
self.assertEqual(receipt["overall"]["status"], "error")
|
|
by_tag = {entry["tag"]: entry for entry in receipt["images"]}
|
|
self.assertEqual(by_tag[VULN_TAG]["status"], "error")
|
|
self.assertEqual(by_tag[CLEAN_TAG]["status"], "pass")
|
|
clean_decision = json.loads(
|
|
Path(by_tag[CLEAN_TAG]["decision"]).read_text(encoding="utf-8")
|
|
)
|
|
self.assertEqual(clean_decision["status"], "pass")
|
|
vuln_decision = json.loads(
|
|
Path(by_tag[VULN_TAG]["decision"]).read_text(encoding="utf-8")
|
|
)
|
|
self.assertEqual(vuln_decision["status"], "error")
|
|
|
|
def test_scanner_version_failure_is_a_bootstrap_error(self):
|
|
result, _, _, receipt, _, _ = self.run_scan(extra_env={"FAKE_TRIVY_VERSION_FAIL": "1"})
|
|
self.assertEqual(result.returncode, 2, result.stderr)
|
|
self.assertEqual(receipt["overall"]["status"], "error")
|
|
|
|
def test_rerun_cannot_reuse_an_old_successful_raw_report(self):
|
|
result, _, evidence, receipt, _, _ = self.run_scan(images=(CLEAN_TAG,), repeat_without_output=True)
|
|
self.assertEqual(result.returncode, 2, result.stderr)
|
|
self.assertEqual(receipt["overall"]["status"], "error")
|
|
self.assertEqual(receipt["images"][0]["status"], "error")
|
|
|
|
def test_bootstrap_failure_writes_error_receipt_for_each_image(self):
|
|
result, _, _, receipt, docker_calls, trivy_calls = self.run_scan(curl_fail=True)
|
|
self.assertEqual(result.returncode, 2, result.stderr)
|
|
self.assertIsNotNone(receipt)
|
|
self.assertEqual(receipt["overall"]["status"], "error")
|
|
self.assertEqual(receipt["overall"]["exit_code"], 2)
|
|
self.assertEqual(len(receipt["images"]), 2)
|
|
self.assertTrue(all(entry["status"] == "error" for entry in receipt["images"]))
|
|
self.assertEqual({entry["tag"] for entry in receipt["images"]}, {VULN_TAG, CLEAN_TAG})
|
|
self.assertEqual(docker_calls, [])
|
|
self.assertEqual(trivy_calls, [])
|
|
|
|
def test_unknown_image_is_error_and_is_never_scanned(self):
|
|
missing = "crank/missing:ci"
|
|
result, _, _, receipt, _, trivy_calls = self.run_scan(images=(missing, CLEAN_TAG))
|
|
self.assertEqual(result.returncode, 2, result.stderr)
|
|
by_tag = {entry["tag"]: entry for entry in receipt["images"]}
|
|
self.assertEqual(by_tag[missing]["status"], "error")
|
|
self.assertEqual(by_tag[CLEAN_TAG]["status"], "pass")
|
|
self.assertEqual([self.target_of(call) for call in self.scan_calls(trivy_calls)], [CLEAN_ID])
|
|
decision = json.loads(Path(by_tag[missing]["decision"]).read_text())
|
|
self.assertEqual(decision["status"], "error")
|
|
|
|
def test_download_error_log_is_retained(self):
|
|
result, _, evidence, receipt, _, _ = self.run_scan(curl_fail=True)
|
|
self.assertEqual(result.returncode, 2, result.stderr)
|
|
self.assertEqual(receipt["overall"]["status"], "error")
|
|
self.assertIn("failed to connect", (evidence / "scanner-download.log").read_text())
|
|
|
|
def test_evidence_kept_even_when_gate_is_red(self):
|
|
_, _, evidence, receipt, _, _ = self.run_scan()
|
|
self.assertTrue((evidence / "receipt.json").is_file())
|
|
self.assertTrue((evidence / "scanner.json").is_file())
|
|
self.assertTrue((evidence / "db" / "metadata.json").is_file())
|
|
scanner = json.loads((evidence / "scanner.json").read_text(encoding="utf-8"))
|
|
self.assertEqual(scanner["expected_version"], "0.70.0")
|
|
self.assertEqual(scanner["policy"]["ignore_unfixed"], False)
|
|
self.assertIsNotNone(receipt)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|