ci: move linux and macos wheel building from GHA to GitLab

.github/workflows/build_deploy.yml

@@ -73,65 +73,3 @@ jobs:
       cibw_build: 'cp39* cp310* cp311* cp312* cp313* cp314*'
       cibw_skip: 'cp39-win_arm64 cp310-win_arm64 cp314t* *_i686'
       library_version: ${{ needs.compute_version.outputs.library_version }}
-
-  build_sdist:
-    needs: [ "compute_version" ]
-    name: Build source distribution
-    runs-on: ubuntu-latest
-    env:
-      SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE: ${{ needs.compute_version.outputs.library_version }}
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        # Include all history and tags
-        with:
-          persist-credentials: false
-          fetch-depth: 0
-      - uses: actions-rust-lang/setup-rust-toolchain@9d7e65c320fdb52dcd45ffaa68deb6c02c8754d9 # v1.12.0
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        name: Install Python
-        with:
-          python-version: '3.12'
-      - name: Build sdist
-        run: |
-          pip install "setuptools_scm[toml]>=4" "cython" "cmake>=3.24.2,<3.28" "setuptools-rust<2"
-          # Disable cython extensions to avoid compiling .pyx files
-          DD_CYTHONIZE=0 python setup.py sdist
-      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        with:
-          name: source-dist
-          path: dist/*.tar.gz
-
-  test_alpine_sdist:
-    # alpine is the only environment not built/tested by cibuildwheel
-    name: Test source distribution on Alpine Linux
-    needs: [build_sdist]
-    runs-on: ubuntu-latest
-    container:
-      image: python:3.9-alpine
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          persist-credentials: false
-      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
-        with:
-          name: source-dist
-          path: dist
-
-      - name: Install build dependencies
-        # Rust + Cargo are needed for Cryptography
-        run: apk add git gcc g++ musl-dev libffi-dev openssl-dev bash rust cargo make cmake patchelf
-
-      - name: Check source package
-        # https://github.com/theacodes/cmarkgfm/issues/84
-        run: |
-          pip install twine readme_renderer[md] "cmarkgfm<2025.10.20"
-          twine check dist/*.tar.gz
-      - name: Install source package
-        env:
-          CMAKE_BUILD_PARALLEL_LEVEL: 12
-        run: pip install dist/*.tar.gz
-
-      - name: Test the source package
-        run: python $GITHUB_WORKSPACE/tests/smoke_test.py
-        # Move out of the workspace to avoid importing ddtrace from the source
-        working-directory: /

.github/workflows/build_python_3.yml

@@ -64,12 +64,8 @@ jobs:
         run: |
           MATRIX_INCLUDE=$(
             {
-              cibuildwheel --print-build-identifiers --platform linux --archs x86_64,i686 | jq -cR '{only: ., os: "ubuntu-latest-16-cores"}' \
-              && cibuildwheel --print-build-identifiers --platform linux --archs aarch64  | jq -cR '{only: ., os: "arm-8core-linux"}' \
-              && cibuildwheel --print-build-identifiers --platform windows --archs AMD64,x86 | jq -cR '{only: ., os: "windows-latest"}' \
-              && cibuildwheel --print-build-identifiers --platform windows --archs ARM64    | jq -cR '{only: ., os: "windows-11-arm"}' \
-              && cibuildwheel --print-build-identifiers --platform macos --archs x86_64 | jq -cR '{only: ., os: "macos-15-large"}' \
-              && cibuildwheel --print-build-identifiers --platform macos --archs arm64 | jq -cR '{only: ., os: "macos-15-xlarge"}'
+              cibuildwheel --print-build-identifiers --platform windows --archs AMD64,x86 | jq -cR '{only: ., os: "windows-latest"}' \
+              && cibuildwheel --print-build-identifiers --platform windows --archs ARM64    | jq -cR '{only: ., os: "windows-11-arm"}'
             } | jq -sc
           )
           echo $MATRIX_INCLUDE
@@ -87,39 +83,12 @@ jobs:
       SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE: ${{ needs.compute_version.outputs.library_version }}
       CIBW_SKIP: ${{ inputs.cibw_skip }}
       CIBW_PRERELEASE_PYTHONS: ${{ inputs.cibw_prerelease_pythons }}
-      CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
-      CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
-      CIBW_MUSLLINUX_I686_IMAGE: ghcr.io/datadog/dd-trace-py/pypa_musllinux_1_2_i686:latest
       CIBW_BEFORE_ALL_WINDOWS: ${{ matrix.os == 'windows-latest' && 'rustup target add i686-pc-windows-msvc' || (matrix.os == 'windows-11-arm' && 'rustup target add aarch64-pc-windows-msvc') }}
-      CIBW_BEFORE_ALL_MACOS: rustup target add aarch64-apple-darwin
-      CIBW_BEFORE_ALL_LINUX: |
-        if [[ "$(uname -m)-$(uname -i)-$(uname -o | tr '[:upper:]' '[:lower:]')-$(ldd --version 2>&1 | head -n 1 | awk '{print $1}')" != "i686-unknown-linux-musl" ]]; then
-          curl -sSf https://sh.rustup.rs | sh -s -- -y;
-        fi
-      CIBW_ENVIRONMENT_LINUX: PATH=$HOME/.cargo/bin:$PATH CMAKE_BUILD_PARALLEL_LEVEL=24 CMAKE_ARGS="-DNATIVE_TESTING=OFF" SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
-      # SYSTEM_VERSION_COMPAT is a workaround for versioning issue, a.k.a.
-      # `platform.mac_ver()` reports incorrect MacOS version at 11.0
-      # See: https://stackoverflow.com/a/65402241
-      CIBW_ENVIRONMENT_MACOS: CMAKE_BUILD_PARALLEL_LEVEL=24 SYSTEM_VERSION_COMPAT=0 CMAKE_ARGS="-DNATIVE_TESTING=OFF" SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
       CIBW_ENVIRONMENT_WINDOWS: SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
       # cibuildwheel repair will copy anything's under /output directory from the
       # build container to the host machine. This is a bit hacky way, but seems
       # to be the only way getting debug symbols out from the container while
       # we don't mess up with RECORD file.
-      CIBW_REPAIR_WHEEL_COMMAND_LINUX: |
-        mkdir -p /output/debugwheelhouse &&
-        python scripts/extract_debug_symbols.py {wheel} --output-dir /output/debugwheelhouse &&
-        python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md &&
-        mkdir ./tempwheelhouse &&
-        unzip -l {wheel} | grep '\.so' &&
-        auditwheel repair -w ./tempwheelhouse {wheel} &&
-        mv ./tempwheelhouse/*.whl {dest_dir} &&
-        rm -rf ./tempwheelhouse
-      CIBW_REPAIR_WHEEL_COMMAND_MACOS: |
-        mkdir -p ./debugwheelhouse &&
-        python scripts/extract_debug_symbols.py {wheel} --output-dir ./debugwheelhouse &&
-        python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md &&
-        MACOSX_DEPLOYMENT_TARGET=12.7 delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
       CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: python scripts/zip_filter.py "{wheel}" "*.c" "*.cpp" "*.cc" "*.h" "*.hpp" "*.pyx" "*.md" && mv "{wheel}" "{dest_dir}"
       CIBW_TEST_COMMAND: "python {project}/tests/smoke_test.py"
 

.gitignore

@@ -48,6 +48,8 @@ var/
 .installed.cfg
 *.egg
 *.whl
+pywheels/
+debugwheelhouse/
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

.gitlab-ci.yml

@@ -137,11 +137,28 @@ run-tests-trigger:
 
 microbenchmarks:
   stage: benchmarks
-  needs: [ "download_ddtrace_artifacts" ]
+  needs:
+    - job: "build linux"
+      parallel:
+        matrix:
+          - ARCH_TAG: "amd64"
+            PYTHON_TAG: "cp39-cp39"
+            IMAGE_TAG: "v85383392-751efc0-manylinux2014_x86_64"
   rules:
+    # Allow failures if explicitly asked
     - if: $RELEASE_ALLOW_BENCHMARK_FAILURES == "true"
       allow_failure: true
-    - allow_failure: false
+    # Always run on PRs, tagged releases, and main or release branches
+    - if: $HAS_OPEN_PR == "true"
+      when: always
+    - if: $IS_MAIN_BRANCH == "true"
+      when: always
+    - if: $IS_RELEASE_BRANCH == "true"
+      when: always
+    - if: $IS_RELEASE == "true"
+      when: always
+    # Otherwise, run manually
+    - when: manual
   trigger:
     include: .gitlab/benchmarks/microbenchmarks.yml
     strategy: depend
@@ -152,19 +169,32 @@ microbenchmarks:
 
 macrobenchmarks:
   stage: benchmarks
-  needs: [ "download_ddtrace_artifacts" ]
+  needs:
+    - job: "build linux"
+      parallel:
+        matrix:
+          - ARCH_TAG: "amd64"
+            PYTHON_TAG: "cp39-cp39"
+            IMAGE_TAG: "v85383392-751efc0-manylinux2014_x86_64"
   trigger:
     include: .gitlab/benchmarks/macrobenchmarks.yml
   variables:
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID
     # Disable VPA for benchmarks
     DD_DISABLE_VPA: true
-  allow_failure: true
+  allow_failure: false
   rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
+    # Allow failures if explicitly asked
+    - if: $RELEASE_ALLOW_BENCHMARK_FAILURES == "true"
+      allow_failure: true
+    # Always run on tagged releases, and main or release branches
+    - if: $IS_MAIN_BRANCH == "true"
+      when: always
+    - if: $IS_RELEASE_BRANCH == "true"
       when: always
     - if: $IS_RELEASE == "true"
       when: always
+    # Otherwise, run manually
     - when: manual
 
 check_new_flaky_tests:

.gitlab/benchmarks/macrobenchmarks.yml

@@ -27,7 +27,8 @@ candidate:
     - interruptible: true
   needs:
     - pipeline: $PARENT_PIPELINE_ID
-      job: download_ddtrace_artifacts
+      job: "build linux: [amd64, cp39-cp39, v85383392-751efc0-manylinux2014_x86_64]"
+      artifacts: true
   script: |
     cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./
     echo "CANDIDATE_WHL=$(ls *.whl | head -n 1)" | tee candidate.env

.gitlab/benchmarks/microbenchmarks.yml

@@ -145,7 +145,8 @@ candidate:
   tags: [ "arch:amd64" ]
   needs:
     - pipeline: $PARENT_PIPELINE_ID
-      job: download_ddtrace_artifacts
+      job: "build linux: [amd64, cp39-cp39, v85383392-751efc0-manylinux2014_x86_64]"
+      artifacts: true
   script: |
     cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./
     echo "CANDIDATE_WHL=$(ls *.whl | head -n 1)" | tee candidate.env
@@ -211,8 +212,8 @@ benchmarks-pr-comment:
     UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for.
 
 # Unlike pre-release SLO checks that run unconditionally (which have
-# "when: always"), this PR-level check runs only if benchmarks succeed. 
-# Since benchmark failures block the PR directly, this job blocks the PR if 
+# "when: always"), this PR-level check runs only if benchmarks succeed.
+# Since benchmark failures block the PR directly, this job blocks the PR if
 # benchmarks succeed but there are SLO breaches.
 check-slo-breaches:
   extends: .check-slo-breaches

.gitlab/package.yml

@@ -1,10 +1,142 @@
+variables:
+  IMAGE_BASE_URL: "registry.ddbuild.io/dd-trace-py"
+
+.PYTHON_VERSIONS: &PYTHON_VERSIONS
+  - "3.9"
+  - "3.10"
+  - "3.11"
+  - "3.12"
+  - "3.13"
+  - "3.14"
+
+.PYTHON_TAGS: &PYTHON_TAGS
+  - "cp39-cp39"
+  - "cp310-cp310"
+  - "cp311-cp311"
+  - "cp312-cp312"
+  - "cp313-cp313"
+  - "cp314-cp314"
+
+.AARCH64_IMAGES: &AARCH64_IMAGES
+  - "v85383414-751efc0-manylinux2014_aarch64"
+  - "v85383359-751efc0-musllinux_1_2_aarch64"
+
+.X86_64_IMAGES: &X86_64_IMAGES
+  - "v85383392-751efc0-manylinux2014_x86_64"
+  - "v85383325-751efc0-musllinux_1_2_x86_64"
+
+.build_base:
+  stage: package
+  variables:
+    CMAKE_BUILD_PARALLEL_LEVEL: "12"
+    CARGO_BUILD_JOBS: "12"
+    CMAKE_ARGS: "-DNATIVE_TESTING=OFF"
+    # DEV: Hack to make ASM CMake able to find the Python3 library
+    CIBW_BUILD: "1"
+    # job resource requests
+    KUBERNETES_CPU_REQUEST: "6"
+    KUBERNETES_MEMORY_REQUEST: "10Gi"
+    KUBERNETES_MEMORY_LIMIT: "10Gi"
+  artifacts:
+    paths:
+      - "pywheels/*.whl"
+      - "pywheels/*.tar.gz"
+
+"build sdist":
+  extends: .build_base
+  image: "${IMAGE_BASE_URL}:v85280965-e932d4a-manylinux2014_x86_64"
+  tags: [ "arch:amd64" ]
+  variables:
+    DD_CYTHONIZE: "0"
+    UV_PYTHON: "/opt/python/cp314-cp314/bin/python"
+  script:
+    - .gitlab/scripts/build-sdist.sh
+
+"build linux":
+  extends: .build_base
+  image: ${IMAGE_BASE_URL}:${IMAGE_TAG}
+  tags: [ "arch:$ARCH_TAG" ]
+  parallel:
+    matrix:
+      - ARCH_TAG: "amd64"
+        PYTHON_TAG: *PYTHON_TAGS
+        IMAGE_TAG: *X86_64_IMAGES
+      - ARCH_TAG: "arm64"
+        PYTHON_TAG: *PYTHON_TAGS
+        IMAGE_TAG: *AARCH64_IMAGES
+  script:
+    - manylinux-interpreters ensure "${PYTHON_TAG}"
+    - export UV_PYTHON="/opt/python/${PYTHON_TAG}/bin/python"
+    - |
+      find src/native -type f -exec touch -t 202401010000 {} \;
+      find ddtrace/ -type f -name "*.pyx" -exec touch -t 202401010000 {} \;
+      find ddtrace/ -type f -name "*.c" -exec touch -t 202401010000 {} \;
+      find ddtrace/ -type f -name "*.cpp" -exec touch -t 202401010000 {} \;
+      find ddtrace/ -type f -name "*.h" -exec touch -t 202401010000 {} \;
+    - .gitlab/scripts/build-wheel.sh
+
+"build macos":
+  extends: .build_base
+  tags: [ "macos:sonoma-$ARCH_TAG" ]
+  parallel:
+    matrix:
+      - ARCH_TAG: "amd64"
+        UV_PYTHON: *PYTHON_VERSIONS
+      - ARCH_TAG: "arm64"
+        UV_PYTHON: *PYTHON_VERSIONS
+  variables:
+    SYSTEM_VERSION_COMPAT: "0"
+  rules:
+    - if: $IS_MAIN_BRANCH == "true" || $IS_RELEASE_BRANCH == "true" || $IS_RELEASE == "true"
+      when: always
+    - if: $UV_PYTHON == "3.14"
+      when: always
+    - when: manual
+  script:
+    - .gitlab/scripts/build-wheel.sh
+
+"test sdist":
+  image: registry.ddbuild.io/images/mirror/python:3.9.6-alpine3.14
+  tags: [ "arch:arm64" ]
+  stage: package
+  variables:
+    CARGO_BUILD_JOBS: "12"
+    CMAKE_BUILD_PARALLEL_LEVEL: "12"
+    KUBERNETES_CPU_REQUEST: "6"
+    KUBERNETES_MEMORY_REQUEST: "10Gi"
+    KUBERNETES_MEMORY_LIMIT: "10Gi"
+  needs:
+    - job: "build sdist"
+      artifacts: true
+  script:
+    - SDIST_FILE=$(ls ${CI_PROJECT_DIR}/pywheels/*.tar.gz | head -n 1)
+    - TMP_DIR=$(mktemp -d) && cd $TMP_DIR
+    - |
+      apk add curl git gcc g++ musl-dev libffi-dev openssl-dev bash cargo make cmake patchelf
+      curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+      export PATH="$HOME/.cargo/bin:$PATH"
+      pip install twine readme_renderer[md] "cmarkgfm<2025.10.20"
+    - twine check ${SDIST_FILE}
+    - pip install ${SDIST_FILE}
+    - python ${CI_PROJECT_DIR}/tests/smoke_test.py
+
 download_ddtrace_artifacts:
   image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
   tags: [ "arch:amd64" ]
   stage: package
   id_tokens:
     DDOCTOSTS_ID_TOKEN:
       aud: dd-octo-sts
+  rules:
+    - if: $IS_MAIN_BRANCH == "true"
+      when: always
+    - if: $IS_RELEASE_BRANCH == "true"
+      when: always
+    - if: $IS_RELEASE == "true"
+      when: always
+    - if: $HAS_OPEN_PR == "true"
+      when: always
+    - when: manual
   script: |
     if [ -z ${GH_TOKEN} ]
     then
@@ -26,7 +158,9 @@ download_dependency_wheels:
   image: registry.ddbuild.io/images/mirror/python:$PYTHON_IMAGE_TAG
   tags: [ "arch:amd64" ]
   stage: package
-  needs: [ download_ddtrace_artifacts ]
+  needs:
+    - job: "build linux"
+      artifacts: true
   variables:
     PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
   parallel:
@@ -58,7 +192,7 @@ publish-wheels-to-s3:
   image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29
   stage: package
   needs:
-    - job: download_ddtrace_artifacts
+    - job: "build linux"
       artifacts: true
   variables:
     BUCKET: dd-trace-py-builds