ci: improve reliability (#232)

JeyJeyGao · web-flow · commit 79fcd26ff3ba · 2025-01-22T15:18:40.000+08:00
Background: as the default `ubuntu-latest` runner [has upgraded](actions/runner-images@976232d) to `ubuntu-24.04`, we need to ensure compatibility with older OS versions if we release the binary on a new runner as the dotnet building process depends on OS libraries (for example, `GLIBC`). CI: - Unit test pipeline - added unit test on Linux arm64 and windows x64. - E2E pipeline - built `linux-x64` binary on `ubuntu-24.04` and tested on `ubuntu-22.04`, `ubuntu-24.04` and `mcr.microsoft.com/cbl-mariner/base/core:2.0` container. - built `linux-arm64` binary on `ubuntu-24.04-arm` and tested on `ubuntu-24.04-arm`, `ubuntu-22.04-arm`. - built `windows-x64` binary on `windows-2022` and tested on `windows-2022` and `windows-2025` (GA). - release pipeline - updated release pipeline to use `ubuntu 24.04` for Linux and macOS binaries and `windows-2022` for Windows binary. - removed ARM emulator from release pipeline and replaced with `ubuntu-24.04-arm` runner. Test: - fixed unit test on windows to avoid line ending symbol issue - tested new release pipeline in forked repo Note: win-x64 build generates a warning which is discussed here: dotnet/runtime#109958. It should be fixed in the future dotnet release. --------- Signed-off-by: Junjie Gao <junjiegao@microsoft.com>
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -17,19 +17,19 @@ jobs:
         config:
           - runtime: osx-x64
             build_args: ""
-            os: ubuntu-latest
+            os: ubuntu-24.04
           - runtime: osx-arm64
             build_args: ""
-            os: ubuntu-latest
+            os: ubuntu-24.04
           - runtime: linux-x64
             build_args: --enable-aot
-            os: ubuntu-latest
+            os: ubuntu-24.04
           - runtime: linux-arm64
             build_args: --enable-aot
-            os: ubuntu-22.04
+            os: ubuntu-24.04-arm
           - runtime: win-x64
             build_args: --enable-aot
-            os: windows-latest
+            os: windows-2022
     name: Build
     runs-on: ${{ matrix.config.os }}
     steps:
@@ -41,19 +41,6 @@ jobs:
         uses: actions/setup-dotnet@v4
         with:
           dotnet-version: '8.0.x'
-      - name: Setup ARM64 Emulator
-        if: matrix.config.runtime == 'linux-arm64'
-        run: |
-          sudo dpkg --add-architecture arm64
-          sudo bash -c 'cat > /etc/apt/sources.list.d/arm64.list <<EOF
-          deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted
-          deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted
-          deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted universe multiverse
-          EOF'
-          sudo sed -i -e 's/deb http/deb [arch=amd64] http/g' /etc/apt/sources.list
-          sudo sed -i -e 's/deb mirror/deb [arch=amd64] mirror/g' /etc/apt/sources.list
-          sudo apt update
-          sudo apt install -y clang llvm binutils-aarch64-linux-gnu gcc-aarch64-linux-gnu zlib1g-dev:arm64
       - name: Build .NET project
         shell: bash
         run: python3 ./scripts/build.py "${GITHUB_REF_NAME}" ${{ matrix.config.runtime }} ${{ matrix.config.build_args }}
@@ -91,7 +78,7 @@ jobs:
           retention-days: 1
   release:
     name: Release
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     needs: macos_codesign
     steps:
       - name: Checkout
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -41,9 +41,9 @@ jobs:
           VALIDATE_MARKDOWN: false
           VALIDATE_JSCPD: false
           VALIDATE_SHELL_SHFMT: false
-  test:
-    name: Unit Testing and Build
-    runs-on: ubuntu-latest
+  test-linux:
+    name: Unit Testing and Build on Linux x64
+    runs-on: ubuntu-24.04
     timeout-minutes: 5
     permissions:
       contents: read
@@ -82,11 +82,63 @@ jobs:
           name: darwin-amd64-binary
           path: ./bin/artifacts/notation-azure-kv_0.0.1_darwin_amd64.tar.gz
           retention-days: 1
+  test-linux-arm:
+    name: Unit Testing and Build on Linux arm64
+    runs-on: ubuntu-24.04-arm
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: '8.0.x'
+      - name: Check out code into the project directory
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Run unit tests
+        run: make test
+      - name: Build Linux Binary
+        run: |
+          # the binary will be used in E2E test
+          python3 ./scripts/build.py v0.0.1 linux-arm64 --enable-aot
+      - name: Upload Linux artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: linux-arm64-binary
+          path: ./bin/artifacts/notation-azure-kv_0.0.1_linux_arm64.tar.gz
+          retention-days: 1
+  test-windows:
+    name: Unit Testing and Build on windows x64
+    runs-on: windows-2022
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: '8.0.x'
+      - name: Check out code into the project directory
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Run unit tests
+        run: make test
+      - name: Build Windows Binary
+        run: python3 ./scripts/build.py v0.0.1 win-x64 --enable-aot
+      - name: Upload Linux artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: windows-amd64-binary
+          path: ./bin/artifacts/notation-azure-kv_0.0.1_windows_amd64.zip
+          retention-days: 1
   e2e-mariner-container:
     name: E2E testing for Mariner container
     runs-on: ubuntu-latest
     environment: E2E
-    needs: test
+    needs: test-linux
     steps:
       - name: Check out code into the project directory
         uses: actions/checkout@v4
@@ -128,24 +180,35 @@ jobs:
           AZURE_FEDERATED_TOKEN_FILE: ./federated_token
   e2e-linux:
     name: E2E testing on Linux
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.config.os }}
+    strategy:
+      matrix:
+        config:
+          - os: ubuntu-24.04
+            arch: amd64
+          - os: ubuntu-22.04
+            arch: amd64
+          - os: ubuntu-24.04-arm
+            arch: arm64
+          - os: ubuntu-22.04-arm
+            arch: arm64
     environment: E2E
-    needs: test
+    needs: [test-linux, test-linux-arm]
     steps:
       - name: Check out code into the project directory
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: actions/download-artifact@v4
         with:
-          name: linux-amd64-binary
+          name: linux-${{ matrix.config.arch }}-binary
           path: ./bin/artifacts
       - name: Run download server locally
         run: |
           nohup python3 -m http.server --directory ./bin/artifacts/ &
 
           # prepare the environment variables for E2E
-          artifactName=notation-azure-kv_0.0.1_linux_amd64.tar.gz
+          artifactName=notation-azure-kv_0.0.1_linux_${{ matrix.config.arch }}.tar.gz
           checksum=$(shasum -a 256 "./bin/artifacts/$artifactName" | awk '{print $1}')
           echo "pluginChecksum=$checksum" >> "$GITHUB_ENV"
           echo "pluginDownloadURL=http://localhost:8000/$artifactName" >> "$GITHUB_ENV"
@@ -168,20 +231,21 @@ jobs:
           pluginChecksum: ${{ env.pluginChecksum }}
   e2e-windows:
     name: E2E testing on Windows
-    runs-on: windows-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-2022, windows-2025]
     environment: E2E
-    needs: test
+    needs: test-windows
     steps:
       - name: Check out code into the project directory
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Setup .NET
-        uses: actions/setup-dotnet@v4
+      - uses: actions/download-artifact@v4
         with:
-          dotnet-version: '8.0.x'
-      - name: Build Windows Binary
-        run: python3 ./scripts/build.py v0.0.1 win-x64 --enable-aot
+          name: windows-amd64-binary
+          path: ./bin/artifacts
       - name: Run download server locally
         run: |
           # wsl bash
@@ -215,7 +279,7 @@ jobs:
     name: E2E testing on macOS
     runs-on: macos-13
     environment: E2E
-    needs: test
+    needs: test-linux
     steps:
       - name: Check out code into the project directory
         uses: actions/checkout@v4
diff --git a/Notation.Plugin.AzureKeyVault.Tests/ProgramTests.cs b/Notation.Plugin.AzureKeyVault.Tests/ProgramTests.cs
@@ -97,10 +97,10 @@ internal interface IResponseMock
 
         [Theory]
         [InlineData(200, "{\"error\":{\"message\":\"TestErrorMessage\"}}", "TestErrorMessage")]
-        [InlineData(500, "{\"error\":{\"message\":\"TestErrorMessage\"}", "Service request failed.\nStatus: 500\n")]
-        [InlineData(500, "{\"error2\":{\"message\":\"TestErrorMessage\"}}", "Service request failed.\nStatus: 500\n")]
-        [InlineData(500, "{\"error\":{\"message2\":\"TestErrorMessage\"}}", "Service request failed.\nStatus: 500\n")]
-        [InlineData(500, "{\"error\":{\"message\":\"\"}}", "\nStatus: 500\n")]
+        [InlineData(500, "{\"error\":{\"message\":\"TestErrorMessage\"}", "Status: 500")]
+        [InlineData(500, "{\"error2\":{\"message\":\"TestErrorMessage\"}}", "Status: 500")]
+        [InlineData(500, "{\"error\":{\"message2\":\"TestErrorMessage\"}}", "Status: 500")]
+        [InlineData(500, "{\"error\":{\"message\":\"\"}}", "Status: 500")]
         public void HandleAzureException(int code, string content, string expectedErrorMessage)
         {
             // Arrange
@@ -124,7 +124,7 @@ public void HandleAzureException(int code, string content, string expectedErrorM
             var errorResponse = Program.HandleAzureException(exception);
 
             // Assert exit code 1
-            Assert.StartsWith(expectedErrorMessage, errorResponse.ErrorMessage);
+            Assert.Contains(expectedErrorMessage, errorResponse.ErrorMessage);
             Assert.Equal("ERROR", errorResponse.ErrorCode);
         }
     }
