Use verified Trivy runner cache and avoid Cargo builds for release-only fixes #118
No files matched your search
@@ -40,6 +40,15 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Detect release-tooling-only changes
|
||||
id: release_tooling
|
||||
env:
|
||||
GITEA_EVENT_NAME: ${{ gitea.event_name }}
|
||||
GITEA_EVENT_BEFORE: ${{ gitea.event.before }}
|
||||
GITEA_PR_BASE_SHA: ${{ gitea.event.pull_request.base.sha }}
|
||||
FORCE_FULL: ${{ inputs.validation_level == 'feature' || contains(gitea.event.pull_request.labels.*.name, 'feature-ready') }}
|
||||
run: python3 scripts/ci-release-tooling-scope.py
|
||||
|
||||
- name: Install Rust toolchain
|
||||
run: |
|
||||
set -eu
|
||||
@@ -78,15 +87,18 @@ jobs:
|
||||
docker info
|
||||
|
||||
- name: Check typed runtime configuration contract
|
||||
if: steps.release_tooling.outputs.ci_only != 'true'
|
||||
run: |
|
||||
cargo run -p crank-config --bin crank-config-contract -- --check
|
||||
python3 scripts/check-runtime-config.py --root .
|
||||
python3 scripts/check-config-boundaries.py --root .
|
||||
|
||||
- name: Check canonical migration contract
|
||||
if: steps.release_tooling.outputs.ci_only != 'true'
|
||||
run: cargo run -p admin-api --bin crank-migrate -- plan --check
|
||||
|
||||
- name: Check typed metrics contract
|
||||
if: steps.release_tooling.outputs.ci_only != 'true'
|
||||
run: |
|
||||
cargo run -p crank-metrics --bin crank-metrics-contract -- --check
|
||||
python3 scripts/check-metrics-boundaries.py --root .
|
||||
@@ -142,6 +154,7 @@ jobs:
|
||||
python3 scripts/check-openapi-matrix-compatibility.py --base "$base_ref"
|
||||
|
||||
- name: Run Crank story ATDD
|
||||
if: steps.release_tooling.outputs.ci_only != 'true'
|
||||
run: scripts/ci-story-checks.sh
|
||||
ui:
|
||||
name: UI Checks
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail-closed detector for release-tooling-only CI changes."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_SHA = re.compile(r"[0-9a-fA-F]{40}\Z")
|
||||
_ALLOWED = {
|
||||
".gitea/workflows/release.yml",
|
||||
"scripts/scan-images.sh",
|
||||
"scripts/check-image-scan.py",
|
||||
"scripts/ci-release-assets.py",
|
||||
"scripts/ci-release-resume.py",
|
||||
"tests/unit/test_scan_images.py",
|
||||
"tests/unit/test_check_image_scan.py",
|
||||
"tests/unit/test_ci_release_assets.py",
|
||||
"tests/unit/test_ci_release_resume.py",
|
||||
}
|
||||
_ALLOWED_BYTES = {path.encode("utf-8") for path in _ALLOWED}
|
||||
|
||||
|
||||
def _git(repo: Path, *args: str) -> subprocess.CompletedProcess[bytes] | None:
|
||||
try:
|
||||
return subprocess.run(
|
||||
["git", "-C", str(repo), *args],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def release_tooling_only(
|
||||
repo: Path, event_name: str, before_sha: str, pr_base_sha: str, force_full: str
|
||||
) -> bool:
|
||||
if force_full.lower() != "false" or event_name not in {"push", "pull_request"}:
|
||||
return False
|
||||
raw_base = before_sha if event_name == "push" else pr_base_sha
|
||||
if not _SHA.fullmatch(raw_base):
|
||||
return False
|
||||
available = _git(repo, "cat-file", "-e", f"{raw_base}^{{commit}}")
|
||||
if available is None or available.returncode != 0:
|
||||
return False
|
||||
if event_name == "pull_request":
|
||||
merge_base = _git(repo, "merge-base", "HEAD", raw_base)
|
||||
if merge_base is None or merge_base.returncode != 0:
|
||||
return False
|
||||
raw_base = merge_base.stdout.decode("ascii", errors="ignore").strip()
|
||||
if not _SHA.fullmatch(raw_base):
|
||||
return False
|
||||
diff = _git(repo, "diff", "--no-renames", "--name-only", "-z", raw_base, "HEAD")
|
||||
if diff is None or diff.returncode != 0:
|
||||
return False
|
||||
paths = diff.stdout.split(b"\0")
|
||||
paths = [path for path in paths if path]
|
||||
return bool(paths) and all(path in _ALLOWED_BYTES for path in paths)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
env = os.environ
|
||||
ci_only = release_tooling_only(
|
||||
Path.cwd(),
|
||||
env.get("GITEA_EVENT_NAME", ""),
|
||||
env.get("GITEA_EVENT_BEFORE", ""),
|
||||
env.get("GITEA_PR_BASE_SHA", ""),
|
||||
env.get("FORCE_FULL", ""),
|
||||
)
|
||||
output = env.get("GITHUB_OUTPUT")
|
||||
if not output:
|
||||
print("Release-tooling CI scope unavailable; using full Rust checks.")
|
||||
return 0
|
||||
with open(output, "a", encoding="utf-8") as stream:
|
||||
stream.write(f"ci_only={'true' if ci_only else 'false'}\n")
|
||||
print("Release-tooling-only change detected." if ci_only else "Full Rust checks required.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+31
-13
@@ -1,8 +1,8 @@
|
||||
#!/bin/sh
|
||||
# Scan the built Crank server images with a pinned Trivy and fail closed.
|
||||
#
|
||||
# The wrapper downloads and checksum-verifies one fixed Trivy release, downloads
|
||||
# one fresh vulnerability database, then scans every requested image by its
|
||||
# The wrapper loads or downloads and checksum-verifies one fixed Trivy release,
|
||||
# downloads one fresh vulnerability database, then scans every requested image by its
|
||||
# immutable Docker id with a fully pinned policy. It never trusts a local
|
||||
# trivy.yaml, .trivyignore, VEX file or skip list: --config and --ignorefile
|
||||
# point at /dev/null and every inherited TRIVY_* variable is removed before
|
||||
@@ -14,6 +14,7 @@
|
||||
#
|
||||
# db/metadata.json copy of the actual downloaded Trivy DB metadata
|
||||
# db-download.log database download log
|
||||
# scanner-cache-source.log archive source provenance
|
||||
# scanner.json pinned scanner version and scan policy receipt
|
||||
# <NN-slug>.json raw Trivy JSON report per image
|
||||
# <NN-slug>.scan.log Trivy scan log per image
|
||||
@@ -47,6 +48,8 @@ expected_sha256="8b4376d5d6befe5c24d503f10ff136d9e0c49f9127a4279fd110b727929a5aa
|
||||
download_url="https://github.com/aquasecurity/trivy/releases/download/v${trivy_version}/${archive}"
|
||||
# Fixed GitHub asset ID for the Trivy 0.70.0 Linux 64-bit release archive.
|
||||
asset_api_url="https://api.github.com/repos/aquasecurity/trivy/releases/assets/398381981"
|
||||
tool_cache_root="${RUNNER_TOOL_CACHE:-/opt/hostedtoolcache}"
|
||||
cached_archive="$tool_cache_root/trivy/$trivy_version/$archive"
|
||||
db_repository_primary="ghcr.io/aquasecurity/trivy-db:2"
|
||||
db_repository_fallback="mirror.gcr.io/aquasec/trivy-db:2"
|
||||
|
||||
@@ -104,19 +107,34 @@ PY
|
||||
exit 2
|
||||
}
|
||||
|
||||
# Download and checksum-verify the pinned scanner exactly once.
|
||||
if ! curl --fail --silent --show-error --location --max-time 300 \
|
||||
"$download_url" --output "$temp_dir/$archive" \
|
||||
>"$output_dir/scanner-download.log" 2>&1; then
|
||||
rm -f "$temp_dir/$archive"
|
||||
# Load the exact-version archive from the runner cache when preinstalled;
|
||||
# otherwise use the bounded network routes. The shared cache is read-only here.
|
||||
cache_source_log="$output_dir/scanner-cache-source.log"
|
||||
: > "$cache_source_log"
|
||||
if [ -f "$cached_archive" ]; then
|
||||
printf 'archive_source=runner-tool-cache\n' > "$cache_source_log"
|
||||
if ! cp "$cached_archive" "$temp_dir/$archive"; then
|
||||
fail_all "could not copy the pinned Trivy archive from runner tool cache"
|
||||
fi
|
||||
else
|
||||
printf 'archive_source=network\n' > "$cache_source_log"
|
||||
if ! curl --fail --silent --show-error --location --max-time 300 \
|
||||
--header 'Accept: application/octet-stream' \
|
||||
"$asset_api_url" --output "$temp_dir/$archive" \
|
||||
>"$output_dir/scanner-api-download.log" 2>&1; then
|
||||
fail_all "could not download pinned Trivy ${trivy_version} from release URL or GitHub asset API"
|
||||
"$download_url" --output "$temp_dir/$archive" \
|
||||
>"$output_dir/scanner-download.log" 2>&1; then
|
||||
rm -f "$temp_dir/$archive"
|
||||
if ! curl --fail --silent --show-error --location --max-time 300 \
|
||||
--header 'Accept: application/octet-stream' \
|
||||
"$asset_api_url" --output "$temp_dir/$archive" \
|
||||
>"$output_dir/scanner-api-download.log" 2>&1; then
|
||||
printf 'network_download=failed\n' >> "$cache_source_log"
|
||||
fail_all "could not download pinned Trivy ${trivy_version} from release URL or GitHub asset API"
|
||||
fi
|
||||
printf 'archive_source=github-release-asset-api\n' > "$cache_source_log"
|
||||
printf 'Downloaded pinned Trivy %s from GitHub release asset API.\n' "$trivy_version" \
|
||||
>>"$output_dir/scanner-api-download.log"
|
||||
else
|
||||
printf 'archive_source=github-release-download\n' > "$cache_source_log"
|
||||
fi
|
||||
printf 'Downloaded pinned Trivy %s from GitHub release asset API.\n' "$trivy_version" \
|
||||
>>"$output_dir/scanner-api-download.log"
|
||||
fi
|
||||
if ! printf '%s %s\n' "$expected_sha256" "$temp_dir/$archive" \
|
||||
| sha256sum --check --status; then
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "ci-release-tooling-scope.py"
|
||||
spec = importlib.util.spec_from_file_location("ci_release_tooling_scope", SCRIPT)
|
||||
assert spec and spec.loader
|
||||
scope = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(scope)
|
||||
|
||||
WORKFLOW = ROOT / ".gitea/workflows/ci.yml"
|
||||
_FALSE = "false"
|
||||
|
||||
|
||||
class ReleaseToolingScopeTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.repo = Path(self.temp.name)
|
||||
self.git("init", "-q")
|
||||
self.git("config", "user.email", "ci-scope@example.test")
|
||||
self.git("config", "user.name", "CI Scope Test")
|
||||
self.write("src/lib.rs", "pub fn initial() {}\n")
|
||||
self.write("scripts/scan-images.sh", "#!/bin/sh\nexit 0\n")
|
||||
self.write("tests/unit/test_scan_images.py", "# initial\n")
|
||||
self.git("add", ".")
|
||||
self.git("commit", "-qm", "baseline")
|
||||
self.base = self.head()
|
||||
|
||||
def git(self, *args: str) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(self.repo), *args],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
def head(self) -> str:
|
||||
return self.git("rev-parse", "HEAD")
|
||||
|
||||
def write(self, relative: str, content: str) -> None:
|
||||
path = self.repo / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
|
||||
def commit(self, message: str = "change") -> None:
|
||||
self.git("add", "-A")
|
||||
self.git("commit", "-qm", message)
|
||||
|
||||
def classify(self, event="push", before=None, base=None, force=_FALSE) -> bool:
|
||||
return scope.release_tooling_only(
|
||||
self.repo,
|
||||
event,
|
||||
self.base if before is None else before,
|
||||
self.base if base is None else base,
|
||||
force,
|
||||
)
|
||||
|
||||
def test_release_only_diff_is_safe_for_push_and_pull_request(self):
|
||||
self.write(".gitea/workflows/release.yml", "# release tooling update\n")
|
||||
self.write("scripts/scan-images.sh", "#!/bin/sh\n# scanner update\n")
|
||||
self.commit()
|
||||
self.assertTrue(self.classify("push", before=self.base))
|
||||
self.assertTrue(self.classify("pull_request", base=self.base))
|
||||
|
||||
def test_rust_or_mixed_diff_requires_full_ci(self):
|
||||
self.write("src/lib.rs", "pub fn changed() {}\n")
|
||||
self.commit("Rust change")
|
||||
self.assertFalse(self.classify("push", before=self.base))
|
||||
|
||||
self.write("scripts/scan-images.sh", "#!/bin/sh\n# tooling change\n")
|
||||
self.write("src/lib.rs", "pub fn changed_again() {}\n")
|
||||
self.commit("mixed change")
|
||||
self.assertFalse(self.classify("push", before=self.base))
|
||||
|
||||
def test_rename_from_rust_into_allowlist_is_not_safe(self):
|
||||
(self.repo / ".gitea/workflows").mkdir(parents=True)
|
||||
self.git("mv", "src/lib.rs", ".gitea/workflows/release.yml")
|
||||
self.commit("rename Rust source into release path")
|
||||
self.assertFalse(self.classify("push", before=self.base))
|
||||
|
||||
def test_unavailable_zero_empty_and_missing_bases_require_full_ci(self):
|
||||
self.write("scripts/scan-images.sh", "#!/bin/sh\n# allowed\n")
|
||||
self.commit()
|
||||
self.assertFalse(self.classify("push", before=""))
|
||||
self.assertFalse(self.classify("push", before="0" * 40))
|
||||
self.assertFalse(self.classify("push", before="1" * 40))
|
||||
self.assertFalse(self.classify("push", before=self.head()))
|
||||
self.assertFalse(self.classify("pull_request", base=""))
|
||||
|
||||
def test_manual_feature_and_unknown_force_metadata_require_full_ci(self):
|
||||
self.write("scripts/scan-images.sh", "#!/bin/sh\n# allowed\n")
|
||||
self.commit()
|
||||
self.assertFalse(self.classify("workflow_dispatch", before=self.base))
|
||||
self.assertFalse(self.classify("push", before=self.base, force="true"))
|
||||
self.assertFalse(self.classify("push", before=self.base, force=""))
|
||||
|
||||
def test_non_allowlisted_ci_self_change_requires_full_ci(self):
|
||||
self.write(".gitea/workflows/ci.yml", "# detector workflow change\n")
|
||||
self.commit()
|
||||
self.assertFalse(self.classify("push", before=self.base))
|
||||
|
||||
|
||||
class WorkflowScopeTests(unittest.TestCase):
|
||||
def test_only_four_heavy_rust_steps_are_skippable(self):
|
||||
workflow = WORKFLOW.read_text(encoding="utf-8")
|
||||
self.assertIn("run: python3 scripts/ci-release-tooling-scope.py", workflow)
|
||||
self.assertIn("FORCE_FULL: ${{ inputs.validation_level == 'feature' || contains(gitea.event.pull_request.labels.*.name, 'feature-ready') }}", workflow)
|
||||
conditional = "if: steps.release_tooling.outputs.ci_only != 'true'"
|
||||
for name in (
|
||||
"Check typed runtime configuration contract",
|
||||
"Check canonical migration contract",
|
||||
"Check typed metrics contract",
|
||||
"Run Crank story ATDD",
|
||||
):
|
||||
start = workflow.index(f"- name: {name}")
|
||||
self.assertIn(conditional, workflow[start : start + 180])
|
||||
for name in (
|
||||
"Install Rust toolchain",
|
||||
"Run tooling unit tests",
|
||||
"Check formatting",
|
||||
"Check Rust boundaries",
|
||||
"Check OpenAPI matrix and corpus compatibility versions",
|
||||
):
|
||||
start = workflow.index(f"- name: {name}")
|
||||
self.assertNotIn("if:", workflow[start : start + 160])
|
||||
self.assertIn("name: Rust Checks", workflow)
|
||||
self.assertIn("name: UI Checks", workflow)
|
||||
self.assertIn("name: Required Checks", workflow)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -189,6 +189,7 @@ class ScanImagesWrapperTests(unittest.TestCase):
|
||||
curl_direct_fail=False,
|
||||
curl_api_fail=False,
|
||||
api_checksum_mismatch=False,
|
||||
cache_archive=None,
|
||||
extra_env=None,
|
||||
workdir_files=None,
|
||||
run_in_workdir=False,
|
||||
@@ -229,6 +230,11 @@ class ScanImagesWrapperTests(unittest.TestCase):
|
||||
archive_path.write_bytes(fixture_archive)
|
||||
api_archive_path = root / "trivy-api-fixture.tar.gz"
|
||||
api_archive_path.write_bytes(fixture_archive + b"tampered" if api_checksum_mismatch else fixture_archive)
|
||||
tool_cache = root / "runner-tool-cache"
|
||||
if cache_archive is not None:
|
||||
cached_path = tool_cache / "trivy" / "0.70.0" / "trivy_0.70.0_Linux-64bit.tar.gz"
|
||||
cached_path.parent.mkdir(parents=True)
|
||||
cached_path.write_bytes(fixture_archive if cache_archive == "valid" else b"invalid archive")
|
||||
|
||||
reports = root / "reports"
|
||||
reports.mkdir()
|
||||
@@ -261,6 +267,7 @@ class ScanImagesWrapperTests(unittest.TestCase):
|
||||
env.update(
|
||||
{
|
||||
"PATH": f"{bin_dir}:{env['PATH']}",
|
||||
"RUNNER_TOOL_CACHE": str(tool_cache),
|
||||
"CRANK_IMAGE_SCAN_OUTPUT_DIR": str(evidence),
|
||||
"FAKE_CURL_ARCHIVE": str(archive_path),
|
||||
"FAKE_CURL_API_ARCHIVE": str(api_archive_path),
|
||||
@@ -493,6 +500,37 @@ class ScanImagesWrapperTests(unittest.TestCase):
|
||||
self.assertIn("checksum mismatch", result.stderr)
|
||||
self.assertEqual(trivy_calls, [])
|
||||
|
||||
def test_preloaded_pinned_archive_runs_without_network(self):
|
||||
result, root, evidence, receipt, _, trivy_calls = self.run_scan(
|
||||
images=(CLEAN_TAG,), cache_archive="valid", curl_fail=True
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(receipt["overall"]["status"], "pass")
|
||||
self.assertEqual(
|
||||
(evidence / "scanner-cache-source.log").read_text(),
|
||||
"archive_source=runner-tool-cache\n",
|
||||
)
|
||||
self.assertFalse((evidence / "scanner-download.log").exists())
|
||||
self.assertFalse((evidence / "scanner-api-download.log").exists())
|
||||
cached = root / "runner-tool-cache" / "trivy" / "0.70.0" / "trivy_0.70.0_Linux-64bit.tar.gz"
|
||||
self.assertEqual(cached.read_bytes(), (root / "trivy-fixture.tar.gz").read_bytes())
|
||||
self.assertTrue(any("--download-db-only" in call["args"] for call in trivy_calls))
|
||||
|
||||
def test_preloaded_archive_hash_mismatch_fails_before_network_or_execution(self):
|
||||
result, _, evidence, receipt, _, trivy_calls = self.run_scan(
|
||||
images=(CLEAN_TAG,), cache_archive="bad", curl_fail=True
|
||||
)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertEqual(receipt["overall"]["status"], "error")
|
||||
self.assertIn("checksum mismatch", result.stderr)
|
||||
self.assertEqual(
|
||||
(evidence / "scanner-cache-source.log").read_text(),
|
||||
"archive_source=runner-tool-cache\n",
|
||||
)
|
||||
self.assertFalse((evidence / "scanner-download.log").exists())
|
||||
self.assertFalse((evidence / "scanner-api-download.log").exists())
|
||||
self.assertEqual(trivy_calls, [])
|
||||
|
||||
def test_unknown_image_is_error_and_is_never_scanned(self):
|
||||
missing = "crank/missing:ci"
|
||||
result, _, _, receipt, _, trivy_calls = self.run_scan(images=(missing, CLEAN_TAG))
|
||||
|
||||
Reference in new issue
Block a user