Merge pull request #68 from linuxserver/use-build-cache-for-sbom-generation

Add commit sha, build number to create a build cache url for buildx s…

Author: GilbN

README.md

@@ -23,7 +23,7 @@ This container is an automated testing tool for Docker images. It's designed to
 
 1.  **Spins up the container:** It runs the target Docker image with a specified tag.
 2.  **Checks for successful startup:** It tails the container's logs, waiting for the `[services.d] done.` message, which confirms the init system has finished and the services are running.
-3.  **Generates an SBOM:** It uses `syft` to create a Software Bill of Materials, providing a complete list of all packages inside the image.
+3.  **Generates an SBOM:** It uses `buildx imagetools inspect` or `syft`(fallback) to create a Software Bill of Materials, providing a complete list of all packages inside the image.
 4.  **Tests the Web UI (optional):** If the container runs a web service, it attempts to connect to the UI and take a screenshot to verify it's accessible and renders correctly.
 5.  **Generates a report:** It gathers all the results—container logs, build info, SBOM, screenshots, and test statuses—into a comprehensive HTML report.
 6.  **Uploads the report (CI only):** In a CI environment, it uploads the final report to an S3 bucket for review.
@@ -115,6 +115,8 @@ sudo docker run --rm -i \
 -e NODE_NAME=<optional, Name of the builder that runs the CI test.> \
 -e RELEASE_TAG=<optional, The release tag of the docker image. Used for upload location. Defaults to 'latest'> \
 -e SYFT_IMAGE_TAG=<optional, The image tag of the syft docker image. Used for generating SBOM. Defaults to '1.26.1'> \
+-e COMMIT_SHA=<commit sha, used for creating the sbom with buildx imagetool inspect> \
+-e BUILD_NUMBER=<jenkins build number, used for creating the sbom with buildx imagetool inspect>
 -t lsiodev/ci:latest \
 python3 test_build.py
 ```

ci/ci.py

@@ -93,6 +93,13 @@ class Platform(enum.Enum):
     RISCV64 = "riscv64"
     UNKNOWN = "amd64"
 
+class BuildCacheTag(enum.Enum):
+    """Enum for the different build cache tags"""
+    AMD64 = "amd64"
+    ARM64 = "arm64v8"
+    RISCV64 = "riscv64"
+    UNKNOWN = "amd64"
+
 class SetEnvs():
     """Simple helper class that sets up the ENVs"""
     def __init__(self) -> None:
@@ -119,6 +126,9 @@ def __init__(self) -> None:
         self.bucket: str = os.environ.get("S3_BUCKET", "ci-tests.linuxserver.io")
         self.release_tag: str = os.environ.get("RELEASE_TAG", "latest")
         self.syft_image_tag: str = os.environ.get("SYFT_IMAGE_TAG", "v1.26.1")
+        self.commit_sha: str = os.environ.get("COMMIT_SHA", "")
+        self.build_number: str = os.environ.get("BUILD_NUMBER", "")
+        self.build_cache_registry: str = os.environ.get("BUILD_CACHE_REGISTRY", "ghcr.io/linuxserver/lsiodev-buildcache")
 
         if os.environ.get("DELAY_START"):
             self.logger.warning("DELAY_START env is obsolete, and not in use anymore")
@@ -169,6 +179,9 @@ def __init__(self) -> None:
         S3_REGION:              '{os.environ.get("S3_REGION")}'
         S3_BUCKET:              '{os.environ.get("S3_BUCKET")}'
         SYFT_IMAGE_TAG:         '{os.environ.get("SYFT_IMAGE_TAG")}'
+        COMMIT_SHA:             '{os.environ.get("COMMIT_SHA")}'
+        BUILD_NUMBER:           '{os.environ.get("BUILD_NUMBER")}'
+        BUILD_CACHE_REGISTRY:   '{os.environ.get("BUILD_CACHE_REGISTRY")}'
         Docker Engine Version:  '{self.get_docker_engine_version()}'
         """)
         self.logger.info(env_data)
@@ -630,6 +643,39 @@ def get_sbom_syft(self, tag: str) -> str | CITestResult:
             self.logger.exception("Failed to remove the Syft container, %s",tag)
         return CITestResult.ERROR
 
+    def get_build_cache_url(self, tag: str) -> str:
+        """Get the build cache URL for the given tag.
+
+        Args:
+            tag (str): The tag we are testing
+        Returns:
+            str: The build cache URL ex `ghcr.io/linuxserver/lsiodev-buildcache:arm64v8-<COMMIT_SHA>-<BUILD_NUMBER>`
+        """
+        platform: str = self.get_build_cache_platform(tag)
+        build_cache_url: str = f"{self.build_cache_registry}:{platform}-{self.commit_sha}-{self.build_number}"
+        return build_cache_url
+
+    def get_build_cache_platform(self, tag: str) -> str:
+        """Get the build cache platform for the given tag.
+
+        Args:
+            tag (str): The tag we are testing
+
+        Returns:
+            str: The build cache platform
+        """
+        
+        platform = self.get_platform(tag)
+        match platform:
+            case Platform.AMD64.value:
+                return BuildCacheTag.AMD64.value
+            case Platform.ARM64.value:
+                return BuildCacheTag.ARM64.value
+            case Platform.RISCV64.value:
+                return BuildCacheTag.RISCV64.value
+            case _:
+                return BuildCacheTag.UNKNOWN.value
+
     def get_sbom_buildx_blob(self, tag: str) -> str | CITestResult:
         """Get the SBOM for the image tag using docker buildx imagetools inspect.
 
@@ -640,7 +686,7 @@ def get_sbom_buildx_blob(self, tag: str) -> str | CITestResult:
             str: SBOM output if successful, otherwise "CITestResult.ERROR".
         """
         try:
-            image_ref = f"{self.image}:{tag}"
+            image_ref:str = self.get_build_cache_url(tag)
             self.logger.info("Generating SBOM for %s using buildx imagetools inspect", image_ref)
             cmd = f'docker buildx imagetools inspect {image_ref} --format "{{{{ json (index .SBOM).SPDX.packages }}}}"'
             result: subprocess.CompletedProcess[str] = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=self.sbom_timeout, check=False)

tests/test_ci.py

@@ -8,7 +8,7 @@
 from docker import DockerClient
 from moto import mock_aws
 
-from ci.ci import CI, SetEnvs, CITestResult, CITests, Platform
+from ci.ci import CI, SetEnvs, CITestResult, CITests, Platform, BuildCacheTag
 
 os.environ["DRY_RUN"] = "false"
 os.environ["IMAGE"] = "linuxserver/test"
@@ -23,6 +23,8 @@
 os.environ["PORT"] = "443"
 os.environ["WEB_SCREENSHOT"] = "true"
 os.environ["WEB_AUTH"] = ""
+os.environ["COMMIT_SHA"] = "test-commit-sha"
+os.environ["BUILD_NUMBER"] = "1234"
 
 @pytest.fixture
 def sbom_blob() -> bytes:
@@ -241,3 +243,13 @@ def test_get_image_name(ci: CI) -> None:
     assert ci.get_image_name() == "linuxserver/lspipepr-plex"
     ci.image = "lsiobase/ubuntu"
     assert ci.get_image_name() == "linuxserver/docker-baseimage-ubuntu"
+
+def test_get_build_cache_url(ci: CI) -> None:
+    for tag in ci.tags:
+        cache_tag = ci.get_build_cache_platform(tag)
+        expected_url = f"{ci.build_cache_registry}:{cache_tag}-{ci.commit_sha}-{ci.build_number}"
+        assert ci.get_build_cache_url(tag) == expected_url
+
+def test_get_build_cache_platform(ci: CI) -> None:
+    assert ci.get_build_cache_platform(ci.tags[0]) == BuildCacheTag.AMD64.value
+    assert ci.get_build_cache_platform(ci.tags[1]) == BuildCacheTag.ARM64.value