From b8de8728061d3a7d8c21c98d5f514ff8db1ceda3 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 25 Nov 2025 08:47:04 -0500 Subject: [PATCH 01/86] POC --- .gitlab/package.yml | 30 ++++++++++++++++++++++++ .gitlab/scripts/build-wheel-manylinux.sh | 21 +++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100755 .gitlab/scripts/build-wheel-manylinux.sh diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 4701a713bc2..874dbf2c4e2 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -1,3 +1,33 @@ +stages: + - package + +.PYTHON_TAGS: &PYTHON_TAGS + - "cp39-cp39" + - "cp310-cp310" + - "cp311-cp311" + - "cp312-cp312" + - "cp313-cp313" + - "cp314-cp314" + +"build linux": + image: registry.ddbuild.io/images/mirror/pypa/manylinux2014_${IMAGE_ARCH}:2025-04-12-5990e2d" + tags: [ "arch:$ARCH_TAG" ] + stage: package + parallel: + matrix: + - IMAGE_ARCH: "x86_64" + ARCH_TAG: "amd64" + PYTHON_TAG: *PYTHON_TAGS + - IMAGE_ARCH: "aarch64" + ARCH_TAG: "arm64" + PYTHON_TAG: *PYTHON_TAGS + variables: + CMAKE_BUILD_PARALLEL_LEVEL: "24" + CARGO_BUILD_JOBS: "24" + CMAKE_ARGS: "-DNATIVE_TESTING=OFF" + script: + - .gitlab/build-wheels-manylinux.sh + download_ddtrace_artifacts: image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 tags: [ "arch:amd64" ] diff --git a/.gitlab/scripts/build-wheel-manylinux.sh b/.gitlab/scripts/build-wheel-manylinux.sh new file mode 100755 index 00000000000..2602eadb1da --- /dev/null +++ b/.gitlab/scripts/build-wheel-manylinux.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +export PYTHON_TAG=cp314-cp314 +export CIBW_BUILD=1 +export CMAKE_ARGS="-DNATIVE_TESTING=OFF" + +# Setup Python environment +manylinux-interpreters ensure "${PYTHON_TAG}" +export PATH="/opt/python/${PYTHON_TAG}/bin:${PATH}" +which python +python --version +which pip +pip --version + +# Install Rust +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +source $HOME/.cargo/env + +# Build wheel +python -m build --wheel --outdir pywheels . From fc0437befc183a524412ff12608259e0cbfa325c Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 25 Nov 2025 10:00:46 -0500 Subject: [PATCH 02/86] make build manylinux script work --- .gitignore | 2 + .gitlab/package.yml | 4 +- .gitlab/scripts/build-wheel-manylinux.sh | 68 ++++++++++++++++++++++-- 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index b4a5c1bf459..5c79ac8728e 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,8 @@ var/ .installed.cfg *.egg *.whl +pywheels/ +debugwheelhouse/ # PyInstaller # Usually these files are written by a python script from a template diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 874dbf2c4e2..c162b916880 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -9,7 +9,7 @@ stages: - "cp313-cp313" - "cp314-cp314" -"build linux": +"build manylinux": image: registry.ddbuild.io/images/mirror/pypa/manylinux2014_${IMAGE_ARCH}:2025-04-12-5990e2d" tags: [ "arch:$ARCH_TAG" ] stage: package @@ -25,6 +25,8 @@ stages: CMAKE_BUILD_PARALLEL_LEVEL: "24" CARGO_BUILD_JOBS: "24" CMAKE_ARGS: "-DNATIVE_TESTING=OFF" + # DEV: Hack to make ASM CMake able to find the Python3 library + CIBW_BUILD: "1" script: - .gitlab/build-wheels-manylinux.sh diff --git a/.gitlab/scripts/build-wheel-manylinux.sh b/.gitlab/scripts/build-wheel-manylinux.sh index 2602eadb1da..fdf45ac79ec 100755 --- a/.gitlab/scripts/build-wheel-manylinux.sh +++ b/.gitlab/scripts/build-wheel-manylinux.sh @@ -1,21 +1,81 @@ #!/usr/bin/env bash set -euo pipefail -export PYTHON_TAG=cp314-cp314 -export CIBW_BUILD=1 -export CMAKE_ARGS="-DNATIVE_TESTING=OFF" +echo -e "\e[0Ksection_start:`date +%s`:setup_env\r\e[0KSetup environment" +PROJECT_DIR="${CI_PROJECT_DIR:-}" +if [ -z "${PROJECT_DIR}" ]; then + # DEV: This scripts dir but up two levels + PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +fi + +BUILT_WHEEL_DIR="/tmp/cibuildwheel/built_wheel" +TMP_WHEEL_DIR="/tmp/cibuildwheel/tmp_wheel" +FINAL_WHEEL_DIR="${PROJECT_DIR}/pywheels" +DEBUG_WHEEL_DIR="${PROJECT_DIR}/debugwheelhouse" +mkdir -p "${BUILT_WHEEL_DIR}" +mkdir -p "${TMP_WHEEL_DIR}" +mkdir -p "${FINAL_WHEEL_DIR}" +mkdir -p "${DEBUG_WHEEL_DIR}" + +echo -e "\e[0Ksection_end:`date +%s`:setup_env\r\e[0K" # Setup Python environment +echo -e "\e[0Ksection_start:`date +%s`:setup_python\r\e[0KSetting up Python ${PYTHON_TAG}" manylinux-interpreters ensure "${PYTHON_TAG}" export PATH="/opt/python/${PYTHON_TAG}/bin:${PATH}" which python python --version which pip pip --version +echo -e "\e[0Ksection_end:`date +%s`:setup_python\r\e[0K" # Install Rust +echo -e "\e[0Ksection_start:`date +%s`:install_rust\r\e[0KInstalling Rust toolchain" curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y source $HOME/.cargo/env +echo -e "\e[0Ksection_end:`date +%s`:install_rust\r\e[0K" # Build wheel -python -m build --wheel --outdir pywheels . +echo -e "\e[0Ksection_start:`date +%s`:build_wheel\r\e[0KBuilding wheel" +python -m build --wheel --outdir "${BUILT_WHEEL_DIR}" . +BUILT_WHEEL_FILE=$(ls ${BUILT_WHEEL_DIR}/*.whl | head -n 1) +echo -e "\e[0Ksection_end:`date +%s`:build_wheel\r\e[0K" + +# Extract debug symbols +echo -e "\e[0Ksection_start:`date +%s`:extract_debug_symbols\r\e[0KExtracting debug symbols" +python scripts/extract_debug_symbols.py "${BUILT_WHEEL_FILE}" --output-dir "${DEBUG_WHEEL_DIR}" +echo -e "\e[0Ksection_end:`date +%s`:extract_debug_symbols\r\e[0K" + +# Strip unneeded files from wheel +echo -e "\e[0Ksection_start:`date +%s`:strip_wheel\r\e[0KStripping unneeded files from wheel" +python scripts/zip_filter.py "${BUILT_WHEEL_FILE}" \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md +echo -e "\e[0Ksection_end:`date +%s`:strip_wheel\r\e[0K" + +# List all .so files in the wheel for verification +echo -e "\e[0Ksection_start:`date +%s`:list_so_files\r\e[0KListing .so files in the wheel" +unzip -l "${BUILT_WHEEL_FILE}" | grep '\.so$' +echo -e "\e[0Ksection_end:`date +%s`:list_so_files\r\e[0K" + +# Repair the wheel +echo -e "\e[0Ksection_start:`date +%s`:repair_wheel\r\e[0KRepairing wheel with auditwheel" +auditwheel repair -w "${TMP_WHEEL_DIR}" "${BUILT_WHEEL_FILE}" +echo -e "\e[0Ksection_end:`date +%s`:repair_wheel\r\e[0K" + +# Move to final resting place +echo -e "\e[0Ksection_start:`date +%s`:finalize_wheel\r\e[0KFinalizing wheel" +TMP_WHEEL_FILE=$(ls ${TMP_WHEEL_DIR}/*.whl | head -n 1) +mv "${TMP_WHEEL_FILE}" "${FINAL_WHEEL_DIR}/" +FINAL_WHEEL_FILE=$(ls ${FINAL_WHEEL_DIR}/*.whl | head -n 1) +echo -e "\e[0Ksection_end:`date +%s`:finalize_wheel\r\e[0K" + +# Test the wheel +echo -e "\e[0Ksection_start:`date +%s`:test_wheel\r\e[0KTesting the wheel" +TEST_WHEEL_DIR="/tmp/test_wheel/" +mkdir -p "${TEST_WHEEL_DIR}" +python -m pip install virtualenv +python -m virtualenv --no-periodic-update --pip=embed --no-setuptools "${TEST_WHEEL_DIR}/venv" +# Install the package +"${TEST_WHEEL_DIR}/venv/bin/pip" install "${FINAL_WHEEL_FILE}" +# Run the smoke tests +cd "${TEST_WHEEL_DIR}" && "${TEST_WHEEL_DIR}/venv/bin/python" "${PROJECT_DIR}/tests/smoke_test.py" +echo -e "\e[0Ksection_end:`date +%s`:test_wheel\r\e[0K" From f141f4d7ce0e0093b7f0e51f6c3c11d5e2b7b9b9 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 25 Nov 2025 10:02:58 -0500 Subject: [PATCH 03/86] remove extra quote --- .gitlab/package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index c162b916880..7509d81707c 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -10,7 +10,7 @@ stages: - "cp314-cp314" "build manylinux": - image: registry.ddbuild.io/images/mirror/pypa/manylinux2014_${IMAGE_ARCH}:2025-04-12-5990e2d" + image: registry.ddbuild.io/images/mirror/pypa/manylinux2014_${IMAGE_ARCH}:2025-04-12-5990e2d tags: [ "arch:$ARCH_TAG" ] stage: package parallel: From f07183a683f23dec85080d42d6a092f1730bc716 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 25 Nov 2025 10:03:18 -0500 Subject: [PATCH 04/86] remove unncessary stage --- .gitlab/package.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 7509d81707c..22a13879fba 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -1,6 +1,3 @@ -stages: - - package - .PYTHON_TAGS: &PYTHON_TAGS - "cp39-cp39" - "cp310-cp310" From f93c3fdc6a575f9e823ef3dbb341b9be43f7cfac Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 11:22:57 -0500 Subject: [PATCH 05/86] use newer manylinux images --- .gitlab/package.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 22a13879fba..59d039e75da 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -1,3 +1,7 @@ +variables: + REGISTRY_BASE_URL: "registry.ddbuild.io/images/mirror/pypa" + IMAGE_TAG: "2025.11.24-1" + .PYTHON_TAGS: &PYTHON_TAGS - "cp39-cp39" - "cp310-cp310" @@ -6,8 +10,12 @@ - "cp313-cp313" - "cp314-cp314" -"build manylinux": - image: registry.ddbuild.io/images/mirror/pypa/manylinux2014_${IMAGE_ARCH}:2025-04-12-5990e2d +.LINUX_OS_IMAGES: &LINUX_OS_IMAGES + - "manylinux2014" + - "musllinux_1_2" + +"build linux": + image: ${REGISTRY_BASE_URL}/${LINUX_OS_IMAGE}_${IMAGE_ARCH}:${IMAGE_TAG} tags: [ "arch:$ARCH_TAG" ] stage: package parallel: @@ -15,9 +23,11 @@ - IMAGE_ARCH: "x86_64" ARCH_TAG: "amd64" PYTHON_TAG: *PYTHON_TAGS + LINUX_OS_IMAGE: *LINUX_OS_IMAGES - IMAGE_ARCH: "aarch64" ARCH_TAG: "arm64" PYTHON_TAG: *PYTHON_TAGS + LINUX_OS_IMAGE: *LINUX_OS_IMAGES variables: CMAKE_BUILD_PARALLEL_LEVEL: "24" CARGO_BUILD_JOBS: "24" @@ -25,7 +35,7 @@ # DEV: Hack to make ASM CMake able to find the Python3 library CIBW_BUILD: "1" script: - - .gitlab/build-wheels-manylinux.sh + - .gitlab/scripts/build-wheels-manylinux.sh download_ddtrace_artifacts: image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 From caf25ce8dbfa11c2ff3565f226edbc0bd679f167 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 11:25:38 -0500 Subject: [PATCH 06/86] fix script name --- .gitlab/package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 59d039e75da..09445847595 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -35,7 +35,7 @@ variables: # DEV: Hack to make ASM CMake able to find the Python3 library CIBW_BUILD: "1" script: - - .gitlab/scripts/build-wheels-manylinux.sh + - .gitlab/scripts/build-wheel-manylinux.sh download_ddtrace_artifacts: image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 From 60ba95c3b4f1decde1c152b43229c4056ec83e78 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 11:59:38 -0500 Subject: [PATCH 07/86] save artifacts --- .gitlab/package.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 09445847595..e8d0edea701 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -36,6 +36,10 @@ variables: CIBW_BUILD: "1" script: - .gitlab/scripts/build-wheel-manylinux.sh + artifacts: + paths: + - "pywheels/*.whl" + - "pywheels/*.tar.gz" download_ddtrace_artifacts: image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 From 7f12ae1a56d6887f2152cd58d762d3903ef45ac8 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 12:13:10 -0500 Subject: [PATCH 08/86] set k8s cpu/mem request --- .gitlab/package.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index e8d0edea701..0275a8a0e39 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -29,11 +29,14 @@ variables: PYTHON_TAG: *PYTHON_TAGS LINUX_OS_IMAGE: *LINUX_OS_IMAGES variables: - CMAKE_BUILD_PARALLEL_LEVEL: "24" - CARGO_BUILD_JOBS: "24" + CMAKE_BUILD_PARALLEL_LEVEL: "12" + CARGO_BUILD_JOBS: "12" CMAKE_ARGS: "-DNATIVE_TESTING=OFF" # DEV: Hack to make ASM CMake able to find the Python3 library CIBW_BUILD: "1" + KUBERNETES_CPU_REQUEST: "6" + KUBERNETES_MEMORY_REQUEST: "4Gi" + KUBERNETES_MEMORY_LIMIT: "4Gi" script: - .gitlab/scripts/build-wheel-manylinux.sh artifacts: From 16958278a31e05c962d7b4ec76bd67ad760a8fe9 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 12:26:14 -0500 Subject: [PATCH 09/86] bump mem request/limit --- .gitlab/package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 0275a8a0e39..24d7372fda7 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -35,8 +35,8 @@ variables: # DEV: Hack to make ASM CMake able to find the Python3 library CIBW_BUILD: "1" KUBERNETES_CPU_REQUEST: "6" - KUBERNETES_MEMORY_REQUEST: "4Gi" - KUBERNETES_MEMORY_LIMIT: "4Gi" + KUBERNETES_MEMORY_REQUEST: "8Gi" + KUBERNETES_MEMORY_LIMIT: "8Gi" script: - .gitlab/scripts/build-wheel-manylinux.sh artifacts: From 845d573be3d88d3eceb9714970901ab0f1d20d13 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 13:26:41 -0500 Subject: [PATCH 10/86] try to set up caches --- .gitlab/package.yml | 12 +++++++++++ .gitlab/scripts/build-wheel-manylinux.sh | 26 +++++++++++++++++++++--- setup.py | 2 +- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 24d7372fda7..d00416ddbc8 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -29,16 +29,28 @@ variables: PYTHON_TAG: *PYTHON_TAGS LINUX_OS_IMAGE: *LINUX_OS_IMAGES variables: + CARGO_HOME: "${CI_PROJECT_DIR}/.cache/cargo" CMAKE_BUILD_PARALLEL_LEVEL: "12" CARGO_BUILD_JOBS: "12" CMAKE_ARGS: "-DNATIVE_TESTING=OFF" # DEV: Hack to make ASM CMake able to find the Python3 library CIBW_BUILD: "1" + DD_USE_SCCACHE: "1" + DD_SETUP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/download_cache" + # Caching + PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip" + SCCACHE_DIR: "${CI_PROJECT_DIR}/.cache/sccache" + SCCACHE_CACHE_SIZE: "2G" + # job resource requests KUBERNETES_CPU_REQUEST: "6" KUBERNETES_MEMORY_REQUEST: "8Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" script: - .gitlab/scripts/build-wheel-manylinux.sh + cache: + - key: v0-build-linux-cache-${ARCH_TAG}-${PYTHON_TAG}-${LINUX_OS_IMAGE} + paths: + - .cache artifacts: paths: - "pywheels/*.whl" diff --git a/.gitlab/scripts/build-wheel-manylinux.sh b/.gitlab/scripts/build-wheel-manylinux.sh index fdf45ac79ec..f8d32eaa86e 100755 --- a/.gitlab/scripts/build-wheel-manylinux.sh +++ b/.gitlab/scripts/build-wheel-manylinux.sh @@ -22,19 +22,35 @@ echo -e "\e[0Ksection_end:`date +%s`:setup_env\r\e[0K" # Setup Python environment echo -e "\e[0Ksection_start:`date +%s`:setup_python\r\e[0KSetting up Python ${PYTHON_TAG}" manylinux-interpreters ensure "${PYTHON_TAG}" -export PATH="/opt/python/${PYTHON_TAG}/bin:${PATH}" +export PATH="/opt/python/${PYTHON_TAG}/bin:${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}" which python python --version which pip pip --version +pip cache info echo -e "\e[0Ksection_end:`date +%s`:setup_python\r\e[0K" # Install Rust echo -e "\e[0Ksection_start:`date +%s`:install_rust\r\e[0KInstalling Rust toolchain" -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -source $HOME/.cargo/env +if ! command -v rustc &> /dev/null; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + source ${CARGO_HOME:-$HOME/.cargo}/env +fi echo -e "\e[0Ksection_end:`date +%s`:install_rust\r\e[0K" +# Install sccache via cargo +echo -e "\e[0Ksection_start:`date +%s`:install_sccache\r\e[0KInstalling sccache" +if ! command -v sccache &> /dev/null; then + if command -v yum &> /dev/null; then + yum install -y openssl-devel + elif command -v apk &> /dev/null; then + apk --no-cache add openssl-dev openssl-libs-static + fi + cargo install sccache +fi +sccache --show-stats +echo -e "\e[0Ksection_end:`date +%s`:install_sccache\r\e[0K" + # Build wheel echo -e "\e[0Ksection_start:`date +%s`:build_wheel\r\e[0KBuilding wheel" python -m build --wheel --outdir "${BUILT_WHEEL_DIR}" . @@ -79,3 +95,7 @@ python -m virtualenv --no-periodic-update --pip=embed --no-setuptools "${TEST_WH # Run the smoke tests cd "${TEST_WHEEL_DIR}" && "${TEST_WHEEL_DIR}/venv/bin/python" "${PROJECT_DIR}/tests/smoke_test.py" echo -e "\e[0Ksection_end:`date +%s`:test_wheel\r\e[0K" + +echo -e "\e[0Ksection_start:`date +%s`:teardown\r\e[0KTearing down" +sccache --show-stats +echo -e "\e[0Ksection_end:`date +%s`:teardown\r diff --git a/setup.py b/setup.py index 1f355a5b1fc..4ca6c5f86c7 100644 --- a/setup.py +++ b/setup.py @@ -345,7 +345,7 @@ def run(self): class LibraryDownload: - CACHE_DIR = HERE / ".download_cache" + CACHE_DIR = Path(os.getenv("DD_SETUP_CACHE_DIR", HERE / ".download_cache")) USE_CACHE = os.getenv("DD_SETUP_CACHE_DOWNLOADS", "1").lower() in ("1", "yes", "on", "true") name = None From 2e21a1f2a48dd26f69fc42bd7c0bb0d29c44557c Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 13:48:48 -0500 Subject: [PATCH 11/86] fix incorrect script --- .gitlab/scripts/build-wheel-manylinux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/scripts/build-wheel-manylinux.sh b/.gitlab/scripts/build-wheel-manylinux.sh index f8d32eaa86e..51035cd99cc 100755 --- a/.gitlab/scripts/build-wheel-manylinux.sh +++ b/.gitlab/scripts/build-wheel-manylinux.sh @@ -98,4 +98,4 @@ echo -e "\e[0Ksection_end:`date +%s`:test_wheel\r\e[0K" echo -e "\e[0Ksection_start:`date +%s`:teardown\r\e[0KTearing down" sccache --show-stats -echo -e "\e[0Ksection_end:`date +%s`:teardown\r +echo -e "\e[0Ksection_end:`date +%s`:teardown\r\e[0K" From 53bd1d4e802548bedcbc76500405aa2d64c78bfd Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 14:32:13 -0500 Subject: [PATCH 12/86] fix path? --- .gitlab/scripts/build-wheel-manylinux.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab/scripts/build-wheel-manylinux.sh b/.gitlab/scripts/build-wheel-manylinux.sh index 51035cd99cc..ed89778aaf4 100755 --- a/.gitlab/scripts/build-wheel-manylinux.sh +++ b/.gitlab/scripts/build-wheel-manylinux.sh @@ -34,7 +34,6 @@ echo -e "\e[0Ksection_end:`date +%s`:setup_python\r\e[0K" echo -e "\e[0Ksection_start:`date +%s`:install_rust\r\e[0KInstalling Rust toolchain" if ! command -v rustc &> /dev/null; then curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - source ${CARGO_HOME:-$HOME/.cargo}/env fi echo -e "\e[0Ksection_end:`date +%s`:install_rust\r\e[0K" From e8ac98636c623fe46886b03148cd4d0fad3599dc Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 14:35:05 -0500 Subject: [PATCH 13/86] display rustc and cargo info --- .gitlab/scripts/build-wheel-manylinux.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab/scripts/build-wheel-manylinux.sh b/.gitlab/scripts/build-wheel-manylinux.sh index ed89778aaf4..4b113684fcf 100755 --- a/.gitlab/scripts/build-wheel-manylinux.sh +++ b/.gitlab/scripts/build-wheel-manylinux.sh @@ -35,6 +35,8 @@ echo -e "\e[0Ksection_start:`date +%s`:install_rust\r\e[0KInstalling Rust toolch if ! command -v rustc &> /dev/null; then curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y fi +which rustc +rustc --version echo -e "\e[0Ksection_end:`date +%s`:install_rust\r\e[0K" # Install sccache via cargo @@ -47,6 +49,8 @@ if ! command -v sccache &> /dev/null; then fi cargo install sccache fi +which sccache +sccache --version sccache --show-stats echo -e "\e[0Ksection_end:`date +%s`:install_sccache\r\e[0K" From dd22e1ca1b36140897aaa4e9308818d05cccebc5 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 14:37:08 -0500 Subject: [PATCH 14/86] gotta set the default rust version --- .gitlab/scripts/build-wheel-manylinux.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/scripts/build-wheel-manylinux.sh b/.gitlab/scripts/build-wheel-manylinux.sh index 4b113684fcf..a03d8cbddbd 100755 --- a/.gitlab/scripts/build-wheel-manylinux.sh +++ b/.gitlab/scripts/build-wheel-manylinux.sh @@ -35,6 +35,7 @@ echo -e "\e[0Ksection_start:`date +%s`:install_rust\r\e[0KInstalling Rust toolch if ! command -v rustc &> /dev/null; then curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y fi +rustup default stable which rustc rustc --version echo -e "\e[0Ksection_end:`date +%s`:install_rust\r\e[0K" From c90ef53ca00dfbaf459b813033341d3b901640f4 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 14:55:34 -0500 Subject: [PATCH 15/86] publish depend on build linux --- .github/workflows/build_python_3.yml | 21 +-------------------- .gitlab/package.yml | 12 ++++++++++++ 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build_python_3.yml b/.github/workflows/build_python_3.yml index 6b718bb4502..ddbd47fcaa2 100644 --- a/.github/workflows/build_python_3.yml +++ b/.github/workflows/build_python_3.yml @@ -64,9 +64,7 @@ jobs: run: | MATRIX_INCLUDE=$( { - cibuildwheel --print-build-identifiers --platform linux --archs x86_64,i686 | jq -cR '{only: ., os: "ubuntu-latest-16-cores"}' \ - && cibuildwheel --print-build-identifiers --platform linux --archs aarch64 | jq -cR '{only: ., os: "arm-8core-linux"}' \ - && cibuildwheel --print-build-identifiers --platform windows --archs AMD64,x86 | jq -cR '{only: ., os: "windows-latest"}' \ + cibuildwheel --print-build-identifiers --platform windows --archs AMD64,x86 | jq -cR '{only: ., os: "windows-latest"}' \ && cibuildwheel --print-build-identifiers --platform windows --archs ARM64 | jq -cR '{only: ., os: "windows-11-arm"}' \ && cibuildwheel --print-build-identifiers --platform macos --archs x86_64 | jq -cR '{only: ., os: "macos-15-large"}' \ && cibuildwheel --print-build-identifiers --platform macos --archs arm64 | jq -cR '{only: ., os: "macos-15-xlarge"}' @@ -87,16 +85,8 @@ jobs: SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE: ${{ needs.compute_version.outputs.library_version }} CIBW_SKIP: ${{ inputs.cibw_skip }} CIBW_PRERELEASE_PYTHONS: ${{ inputs.cibw_prerelease_pythons }} - CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 - CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014 - CIBW_MUSLLINUX_I686_IMAGE: ghcr.io/datadog/dd-trace-py/pypa_musllinux_1_2_i686:latest CIBW_BEFORE_ALL_WINDOWS: ${{ matrix.os == 'windows-latest' && 'rustup target add i686-pc-windows-msvc' || (matrix.os == 'windows-11-arm' && 'rustup target add aarch64-pc-windows-msvc') }} CIBW_BEFORE_ALL_MACOS: rustup target add aarch64-apple-darwin - CIBW_BEFORE_ALL_LINUX: | - if [[ "$(uname -m)-$(uname -i)-$(uname -o | tr '[:upper:]' '[:lower:]')-$(ldd --version 2>&1 | head -n 1 | awk '{print $1}')" != "i686-unknown-linux-musl" ]]; then - curl -sSf https://sh.rustup.rs | sh -s -- -y; - fi - CIBW_ENVIRONMENT_LINUX: PATH=$HOME/.cargo/bin:$PATH CMAKE_BUILD_PARALLEL_LEVEL=24 CMAKE_ARGS="-DNATIVE_TESTING=OFF" SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }} # SYSTEM_VERSION_COMPAT is a workaround for versioning issue, a.k.a. # `platform.mac_ver()` reports incorrect MacOS version at 11.0 # See: https://stackoverflow.com/a/65402241 @@ -106,15 +96,6 @@ jobs: # build container to the host machine. This is a bit hacky way, but seems # to be the only way getting debug symbols out from the container while # we don't mess up with RECORD file. - CIBW_REPAIR_WHEEL_COMMAND_LINUX: | - mkdir -p /output/debugwheelhouse && - python scripts/extract_debug_symbols.py {wheel} --output-dir /output/debugwheelhouse && - python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md && - mkdir ./tempwheelhouse && - unzip -l {wheel} | grep '\.so' && - auditwheel repair -w ./tempwheelhouse {wheel} && - mv ./tempwheelhouse/*.whl {dest_dir} && - rm -rf ./tempwheelhouse CIBW_REPAIR_WHEEL_COMMAND_MACOS: | mkdir -p ./debugwheelhouse && python scripts/extract_debug_symbols.py {wheel} --output-dir ./debugwheelhouse && diff --git a/.gitlab/package.yml b/.gitlab/package.yml index d00416ddbc8..d4fcbcd2bf1 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -118,6 +118,18 @@ publish-wheels-to-s3: needs: - job: download_ddtrace_artifacts artifacts: true + - job: "build linux" + artifacts: true + parallel: + matrix: + - IMAGE_ARCH: "x86_64" + ARCH_TAG: "amd64" + PYTHON_TAG: *PYTHON_TAGS + LINUX_OS_IMAGE: *LINUX_OS_IMAGES + - IMAGE_ARCH: "aarch64" + ARCH_TAG: "arm64" + PYTHON_TAG: *PYTHON_TAGS + LINUX_OS_IMAGE: *LINUX_OS_IMAGES variables: BUCKET: dd-trace-py-builds script: From e8bcf1067ef43c42230125906c7904836c677e58 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 14:58:38 -0500 Subject: [PATCH 16/86] update needs --- .gitlab/package.yml | 52 ++++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index d4fcbcd2bf1..e36afe1aabd 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -20,14 +20,14 @@ variables: stage: package parallel: matrix: - - IMAGE_ARCH: "x86_64" - ARCH_TAG: "amd64" - PYTHON_TAG: *PYTHON_TAGS - LINUX_OS_IMAGE: *LINUX_OS_IMAGES - - IMAGE_ARCH: "aarch64" - ARCH_TAG: "arm64" - PYTHON_TAG: *PYTHON_TAGS - LINUX_OS_IMAGE: *LINUX_OS_IMAGES + - IMAGE_ARCH: "x86_64" + ARCH_TAG: "amd64" + PYTHON_TAG: *PYTHON_TAGS + LINUX_OS_IMAGE: *LINUX_OS_IMAGES + - IMAGE_ARCH: "aarch64" + ARCH_TAG: "arm64" + PYTHON_TAG: *PYTHON_TAGS + LINUX_OS_IMAGE: *LINUX_OS_IMAGES variables: CARGO_HOME: "${CI_PROJECT_DIR}/.cache/cargo" CMAKE_BUILD_PARALLEL_LEVEL: "12" @@ -84,7 +84,21 @@ download_dependency_wheels: image: registry.ddbuild.io/images/mirror/python:$PYTHON_IMAGE_TAG tags: [ "arch:amd64" ] stage: package - needs: [ download_ddtrace_artifacts ] + needs: + - job: download_ddtrace_artifacts + artifacts: true + - job: "build linux" + artifacts: true + parallel: + matrix: + - IMAGE_ARCH: "x86_64" + ARCH_TAG: "amd64" + PYTHON_TAG: *PYTHON_TAGS + LINUX_OS_IMAGE: *LINUX_OS_IMAGES + - IMAGE_ARCH: "aarch64" + ARCH_TAG: "arm64" + PYTHON_TAG: *PYTHON_TAGS + LINUX_OS_IMAGE: *LINUX_OS_IMAGES variables: PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip" parallel: @@ -116,20 +130,20 @@ publish-wheels-to-s3: image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29 stage: package needs: - - job: download_ddtrace_artifacts - artifacts: true + # - job: download_ddtrace_artifacts + # artifacts: true - job: "build linux" artifacts: true parallel: matrix: - - IMAGE_ARCH: "x86_64" - ARCH_TAG: "amd64" - PYTHON_TAG: *PYTHON_TAGS - LINUX_OS_IMAGE: *LINUX_OS_IMAGES - - IMAGE_ARCH: "aarch64" - ARCH_TAG: "arm64" - PYTHON_TAG: *PYTHON_TAGS - LINUX_OS_IMAGE: *LINUX_OS_IMAGES + - IMAGE_ARCH: "x86_64" + ARCH_TAG: "amd64" + PYTHON_TAG: *PYTHON_TAGS + LINUX_OS_IMAGE: *LINUX_OS_IMAGES + - IMAGE_ARCH: "aarch64" + ARCH_TAG: "arm64" + PYTHON_TAG: *PYTHON_TAGS + LINUX_OS_IMAGE: *LINUX_OS_IMAGES variables: BUCKET: dd-trace-py-builds script: From c5be7828dfcf24ea31d166db999f3b14a725c1ac Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 15:08:05 -0500 Subject: [PATCH 17/86] start microbenchmarks as soon as manylinux2014 x86_64 cp39-cp39 wheel is available --- .gitlab-ci.yml | 9 ++++++++- .gitlab/benchmarks/microbenchmarks.yml | 8 +++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 08297bf612e..4f60398042c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -92,7 +92,14 @@ run-tests-trigger: microbenchmarks: stage: benchmarks - needs: [ "download_ddtrace_artifacts" ] + needs: + - job: build linux + parallel: + matrix: + - IMAGE_ARCH: "x86_64" + ARCH_TAG: "amd64" + PYTHON_TAG: "cp30-cp39" + LINUX_OS_IMAGE: "manylinux2014" rules: - if: $RELEASE_ALLOW_BENCHMARK_FAILURES == "true" allow_failure: true diff --git a/.gitlab/benchmarks/microbenchmarks.yml b/.gitlab/benchmarks/microbenchmarks.yml index 4df7dc040c7..05aa5d54e6c 100644 --- a/.gitlab/benchmarks/microbenchmarks.yml +++ b/.gitlab/benchmarks/microbenchmarks.yml @@ -138,7 +138,13 @@ candidate: tags: [ "arch:amd64" ] needs: - pipeline: $PARENT_PIPELINE_ID - job: download_ddtrace_artifacts + job: build linux + parallel: + matrix: + - IMAGE_ARCH: "x86_64" + ARCH_TAG: "amd64" + PYTHON_TAG: "cp30-cp39" + LINUX_OS_IMAGE: "manylinux2014" script: | cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./ echo "CANDIDATE_WHL=$(ls *.whl | head -n 1)" | tee candidate.env From 3e985769276b2019c18d3962f7377250fd395cbf Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 15:10:43 -0500 Subject: [PATCH 18/86] update macrobenchmark trigger --- .gitlab-ci.yml | 9 ++++++++- .gitlab/benchmarks/macrobenchmarks.yml | 8 +++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4f60398042c..049fdca0b8c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -114,7 +114,14 @@ microbenchmarks: macrobenchmarks: stage: benchmarks - needs: [ "download_ddtrace_artifacts" ] + needs: + - job: build linux + parallel: + matrix: + - IMAGE_ARCH: "x86_64" + ARCH_TAG: "amd64" + PYTHON_TAG: "cp30-cp39" + LINUX_OS_IMAGE: "manylinux2014" trigger: include: .gitlab/benchmarks/macrobenchmarks.yml variables: diff --git a/.gitlab/benchmarks/macrobenchmarks.yml b/.gitlab/benchmarks/macrobenchmarks.yml index 26d035fd9f3..c7cb110d582 100644 --- a/.gitlab/benchmarks/macrobenchmarks.yml +++ b/.gitlab/benchmarks/macrobenchmarks.yml @@ -23,7 +23,13 @@ candidate: - interruptible: true needs: - pipeline: $PARENT_PIPELINE_ID - job: download_ddtrace_artifacts + job: build linux + parallel: + matrix: + - IMAGE_ARCH: "x86_64" + ARCH_TAG: "amd64" + PYTHON_TAG: "cp30-cp39" + LINUX_OS_IMAGE: "manylinux2014" script: | cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./ echo "CANDIDATE_WHL=$(ls *.whl | head -n 1)" | tee candidate.env From e2f68693bd81d8bb33c5b8f5df52b2da526c8619 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 15:32:36 -0500 Subject: [PATCH 19/86] try moving over macos builds --- .github/workflows/build_python_3.yml | 12 --- .gitlab-ci.yml | 8 +- .gitlab/benchmarks/macrobenchmarks.yml | 2 +- .gitlab/benchmarks/microbenchmarks.yml | 2 +- .gitlab/package.yml | 69 +++++++++++---- .gitlab/scripts/build-wheel-macos.sh | 102 +++++++++++++++++++++++ .gitlab/scripts/build-wheel-manylinux.sh | 4 +- 7 files changed, 162 insertions(+), 37 deletions(-) create mode 100755 .gitlab/scripts/build-wheel-macos.sh diff --git a/.github/workflows/build_python_3.yml b/.github/workflows/build_python_3.yml index ddbd47fcaa2..51c05eedeee 100644 --- a/.github/workflows/build_python_3.yml +++ b/.github/workflows/build_python_3.yml @@ -66,8 +66,6 @@ jobs: { cibuildwheel --print-build-identifiers --platform windows --archs AMD64,x86 | jq -cR '{only: ., os: "windows-latest"}' \ && cibuildwheel --print-build-identifiers --platform windows --archs ARM64 | jq -cR '{only: ., os: "windows-11-arm"}' \ - && cibuildwheel --print-build-identifiers --platform macos --archs x86_64 | jq -cR '{only: ., os: "macos-15-large"}' \ - && cibuildwheel --print-build-identifiers --platform macos --archs arm64 | jq -cR '{only: ., os: "macos-15-xlarge"}' } | jq -sc ) echo $MATRIX_INCLUDE @@ -86,21 +84,11 @@ jobs: CIBW_SKIP: ${{ inputs.cibw_skip }} CIBW_PRERELEASE_PYTHONS: ${{ inputs.cibw_prerelease_pythons }} CIBW_BEFORE_ALL_WINDOWS: ${{ matrix.os == 'windows-latest' && 'rustup target add i686-pc-windows-msvc' || (matrix.os == 'windows-11-arm' && 'rustup target add aarch64-pc-windows-msvc') }} - CIBW_BEFORE_ALL_MACOS: rustup target add aarch64-apple-darwin - # SYSTEM_VERSION_COMPAT is a workaround for versioning issue, a.k.a. - # `platform.mac_ver()` reports incorrect MacOS version at 11.0 - # See: https://stackoverflow.com/a/65402241 - CIBW_ENVIRONMENT_MACOS: CMAKE_BUILD_PARALLEL_LEVEL=24 SYSTEM_VERSION_COMPAT=0 CMAKE_ARGS="-DNATIVE_TESTING=OFF" SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }} CIBW_ENVIRONMENT_WINDOWS: SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }} # cibuildwheel repair will copy anything's under /output directory from the # build container to the host machine. This is a bit hacky way, but seems # to be the only way getting debug symbols out from the container while # we don't mess up with RECORD file. - CIBW_REPAIR_WHEEL_COMMAND_MACOS: | - mkdir -p ./debugwheelhouse && - python scripts/extract_debug_symbols.py {wheel} --output-dir ./debugwheelhouse && - python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md && - MACOSX_DEPLOYMENT_TARGET=12.7 delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: python scripts/zip_filter.py "{wheel}" "*.c" "*.cpp" "*.cc" "*.h" "*.hpp" "*.pyx" "*.md" && mv "{wheel}" "{dest_dir}" CIBW_TEST_COMMAND: "python {project}/tests/smoke_test.py" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 049fdca0b8c..1698d742131 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,12 +93,12 @@ run-tests-trigger: microbenchmarks: stage: benchmarks needs: - - job: build linux + - job: "build linux" parallel: matrix: - IMAGE_ARCH: "x86_64" ARCH_TAG: "amd64" - PYTHON_TAG: "cp30-cp39" + PYTHON_TAG: "cp39-cp39" LINUX_OS_IMAGE: "manylinux2014" rules: - if: $RELEASE_ALLOW_BENCHMARK_FAILURES == "true" @@ -115,12 +115,12 @@ microbenchmarks: macrobenchmarks: stage: benchmarks needs: - - job: build linux + - job: "build linux" parallel: matrix: - IMAGE_ARCH: "x86_64" ARCH_TAG: "amd64" - PYTHON_TAG: "cp30-cp39" + PYTHON_TAG: "cp39-cp39" LINUX_OS_IMAGE: "manylinux2014" trigger: include: .gitlab/benchmarks/macrobenchmarks.yml diff --git a/.gitlab/benchmarks/macrobenchmarks.yml b/.gitlab/benchmarks/macrobenchmarks.yml index c7cb110d582..e5b99c810fa 100644 --- a/.gitlab/benchmarks/macrobenchmarks.yml +++ b/.gitlab/benchmarks/macrobenchmarks.yml @@ -28,7 +28,7 @@ candidate: matrix: - IMAGE_ARCH: "x86_64" ARCH_TAG: "amd64" - PYTHON_TAG: "cp30-cp39" + PYTHON_TAG: "cp39-cp39" LINUX_OS_IMAGE: "manylinux2014" script: | cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./ diff --git a/.gitlab/benchmarks/microbenchmarks.yml b/.gitlab/benchmarks/microbenchmarks.yml index 05aa5d54e6c..1561af40fe3 100644 --- a/.gitlab/benchmarks/microbenchmarks.yml +++ b/.gitlab/benchmarks/microbenchmarks.yml @@ -143,7 +143,7 @@ candidate: matrix: - IMAGE_ARCH: "x86_64" ARCH_TAG: "amd64" - PYTHON_TAG: "cp30-cp39" + PYTHON_TAG: "cp39-cp39" LINUX_OS_IMAGE: "manylinux2014" script: | cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./ diff --git a/.gitlab/package.yml b/.gitlab/package.yml index e36afe1aabd..bffa49aa0c8 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -2,6 +2,14 @@ variables: REGISTRY_BASE_URL: "registry.ddbuild.io/images/mirror/pypa" IMAGE_TAG: "2025.11.24-1" +.PYTHON_VERSIONS: &PYTHON_VERSIONS + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" + .PYTHON_TAGS: &PYTHON_TAGS - "cp39-cp39" - "cp310-cp310" @@ -14,20 +22,8 @@ variables: - "manylinux2014" - "musllinux_1_2" -"build linux": - image: ${REGISTRY_BASE_URL}/${LINUX_OS_IMAGE}_${IMAGE_ARCH}:${IMAGE_TAG} - tags: [ "arch:$ARCH_TAG" ] +.build_base: stage: package - parallel: - matrix: - - IMAGE_ARCH: "x86_64" - ARCH_TAG: "amd64" - PYTHON_TAG: *PYTHON_TAGS - LINUX_OS_IMAGE: *LINUX_OS_IMAGES - - IMAGE_ARCH: "aarch64" - ARCH_TAG: "arm64" - PYTHON_TAG: *PYTHON_TAGS - LINUX_OS_IMAGE: *LINUX_OS_IMAGES variables: CARGO_HOME: "${CI_PROJECT_DIR}/.cache/cargo" CMAKE_BUILD_PARALLEL_LEVEL: "12" @@ -39,22 +35,61 @@ variables: DD_SETUP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/download_cache" # Caching PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip" + UV_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/uv" + UV_INSTALL_DIR: "${CI_PROJECT_DIR}/.uv/" SCCACHE_DIR: "${CI_PROJECT_DIR}/.cache/sccache" SCCACHE_CACHE_SIZE: "2G" # job resource requests KUBERNETES_CPU_REQUEST: "6" KUBERNETES_MEMORY_REQUEST: "8Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" + artifacts: + paths: + - "pywheels/*.whl" + - "pywheels/*.tar.gz" + +"build linux": + extends: .build_base + image: ${REGISTRY_BASE_URL}/${LINUX_OS_IMAGE}_${IMAGE_ARCH}:${IMAGE_TAG} + tags: [ "arch:$ARCH_TAG" ] + parallel: + matrix: + - IMAGE_ARCH: "x86_64" + ARCH_TAG: "amd64" + PYTHON_TAG: *PYTHON_TAGS + LINUX_OS_IMAGE: *LINUX_OS_IMAGES + - IMAGE_ARCH: "aarch64" + ARCH_TAG: "arm64" + PYTHON_TAG: *PYTHON_TAGS + LINUX_OS_IMAGE: *LINUX_OS_IMAGES script: - .gitlab/scripts/build-wheel-manylinux.sh cache: - key: v0-build-linux-cache-${ARCH_TAG}-${PYTHON_TAG}-${LINUX_OS_IMAGE} paths: - .cache - artifacts: - paths: - - "pywheels/*.whl" - - "pywheels/*.tar.gz" + - .uv + +"build macos": + extends: .build_base + tags: [ "macos:sonoma-$ARCH_TAG" ] + parallel: + matrix: + - IMAGE_ARCH: "x86_64" + ARCH_TAG: "amd64" + PYTHON_VERSION: *PYTHON_VERSIONS + - IMAGE_ARCH: "aarch64" + ARCH_TAG: "arm64" + PYTHON_VERSION: *PYTHON_VERSIONS + variables: + SYSTEM_VERSION_COMPAT: "0" + script: + - .gitlab/scripts/build-wheel-macos.sh + cache: + - key: v0-build-macos-cache-${ARCH_TAG}-${PYTHON_TAG} + paths: + - .cache + - .uv download_ddtrace_artifacts: image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 diff --git a/.gitlab/scripts/build-wheel-macos.sh b/.gitlab/scripts/build-wheel-macos.sh new file mode 100755 index 00000000000..33eb6190466 --- /dev/null +++ b/.gitlab/scripts/build-wheel-macos.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo -e "\e[0Ksection_start:`date +%s`:setup_env\r\e[0KSetup environment" +PROJECT_DIR="${CI_PROJECT_DIR:-}" +if [ -z "${PROJECT_DIR}" ]; then + # DEV: This scripts dir but up two levels + PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +fi + +BUILT_WHEEL_DIR="/tmp/cibuildwheel/built_wheel" +TMP_WHEEL_DIR="/tmp/cibuildwheel/tmp_wheel" +FINAL_WHEEL_DIR="${PROJECT_DIR}/pywheels" +DEBUG_WHEEL_DIR="${PROJECT_DIR}/debugwheelhouse" +mkdir -p "${BUILT_WHEEL_DIR}" +mkdir -p "${TMP_WHEEL_DIR}" +mkdir -p "${FINAL_WHEEL_DIR}" +mkdir -p "${DEBUG_WHEEL_DIR}" +echo -e "\e[0Ksection_end:`date +%s`:setup_env\r\e[0K" + +# Setup Python environment +echo -e "\e[0Ksection_start:`date +%s`:setup_python\r\e[0KSetting up Python ${PYTHON_VERSION}" +export PATH="${HOME}/.uv/bin/:${PATH}" +if ! command -v uv &> /dev/null; then + curl -LsSf https://astral.sh/uv/install.sh | sh +fi +uv python install "${PYTHON_VERSION}" +which python +python --version +which pip +pip --version +pip cache info +echo -e "\e[0Ksection_end:`date +%s`:setup_python\r\e[0K" + +echo -e "\e[0Ksection_start:`date +%s`:install_rust\r\e[0KInstalling Rust toolchain" +export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}" +rustup update stable +rustup default stable +rustup target add aarch64-apple-darwin +which rustc +rustc --version +echo -e "\e[0Ksection_end:`date +%s`:install_rust\r\e[0K" + +# Install sccache via cargo +echo -e "\e[0Ksection_start:`date +%s`:install_sccache\r\e[0KInstalling sccache" +if ! command -v sccache &> /dev/null; then + cargo install sccache +fi +which sccache +sccache --version +sccache --show-stats +echo -e "\e[0Ksection_end:`date +%s`:install_sccache\r\e[0K" + +# Build wheel +echo -e "\e[0Ksection_start:`date +%s`:build_wheel\r\e[0KBuilding wheel" +python -m build --wheel --outdir "${BUILT_WHEEL_DIR}" . +BUILT_WHEEL_FILE=$(ls ${BUILT_WHEEL_DIR}/*.whl | head -n 1) +echo -e "\e[0Ksection_end:`date +%s`:build_wheel\r\e[0K" + + +# Extract debug symbols +echo -e "\e[0Ksection_start:`date +%s`:extract_debug_symbols\r\e[0KExtracting debug symbols" +python scripts/extract_debug_symbols.py "${BUILT_WHEEL_FILE}" --output-dir "${DEBUG_WHEEL_DIR}" +echo -e "\e[0Ksection_end:`date +%s`:extract_debug_symbols\r\e[0K" + +# Strip unneeded files from wheel +echo -e "\e[0Ksection_start:`date +%s`:strip_wheel\r\e[0KStripping unneeded files from wheel" +python scripts/zip_filter.py "${BUILT_WHEEL_FILE}" \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md +echo -e "\e[0Ksection_end:`date +%s`:strip_wheel\r\e[0K" + +# List all .so files in the wheel for verification +echo -e "\e[0Ksection_start:`date +%s`:list_so_files\r\e[0KListing .so files in the wheel" +unzip -l "${BUILT_WHEEL_FILE}" | grep '\.so$' +echo -e "\e[0Ksection_end:`date +%s`:list_so_files\r\e[0K" + +# Repair the wheel +echo -e "\e[0Ksection_start:`date +%s`:repair_wheel\r\e[0KRepairing wheel with delocate-wheel" +MACOSX_DEPLOYMENT_TARGET=12.7 delocate-wheel --require-archs "${ARCH_TAG}" -w "${TMP_WHEEL_DIR}" -v "${BUILT_WHEEL_FILE}" +echo -e "\e[0Ksection_end:`date +%s`:repair_wheel\r\e[0K" + +# Move to final resting place +echo -e "\e[0Ksection_start:`date +%s`:finalize_wheel\r\e[0KFinalizing wheel" +TMP_WHEEL_FILE=$(ls ${TMP_WHEEL_DIR}/*.whl | head -n 1) +mv "${TMP_WHEEL_FILE}" "${FINAL_WHEEL_DIR}/" +FINAL_WHEEL_FILE=$(ls ${FINAL_WHEEL_DIR}/*.whl | head -n 1) +echo -e "\e[0Ksection_end:`date +%s`:finalize_wheel\r\e[0K" + +# Test the wheel +echo -e "\e[0Ksection_start:`date +%s`:test_wheel\r\e[0KTesting the wheel" +TEST_WHEEL_DIR="/tmp/test_wheel/" +mkdir -p "${TEST_WHEEL_DIR}" +python -m pip install virtualenv +python -m virtualenv --no-periodic-update --pip=embed --no-setuptools "${TEST_WHEEL_DIR}/venv" +# Install the package +"${TEST_WHEEL_DIR}/venv/bin/pip" install "${FINAL_WHEEL_FILE}" +# Run the smoke tests +cd "${TEST_WHEEL_DIR}" && "${TEST_WHEEL_DIR}/venv/bin/python" "${PROJECT_DIR}/tests/smoke_test.py" +echo -e "\e[0Ksection_end:`date +%s`:test_wheel\r\e[0K" + +echo -e "\e[0Ksection_start:`date +%s`:teardown\r\e[0KTearing down" +sccache --show-stats +echo -e "\e[0Ksection_end:`date +%s`:teardown\r\e[0K" diff --git a/.gitlab/scripts/build-wheel-manylinux.sh b/.gitlab/scripts/build-wheel-manylinux.sh index a03d8cbddbd..60c29a2d7d9 100755 --- a/.gitlab/scripts/build-wheel-manylinux.sh +++ b/.gitlab/scripts/build-wheel-manylinux.sh @@ -16,13 +16,12 @@ mkdir -p "${BUILT_WHEEL_DIR}" mkdir -p "${TMP_WHEEL_DIR}" mkdir -p "${FINAL_WHEEL_DIR}" mkdir -p "${DEBUG_WHEEL_DIR}" - echo -e "\e[0Ksection_end:`date +%s`:setup_env\r\e[0K" # Setup Python environment echo -e "\e[0Ksection_start:`date +%s`:setup_python\r\e[0KSetting up Python ${PYTHON_TAG}" manylinux-interpreters ensure "${PYTHON_TAG}" -export PATH="/opt/python/${PYTHON_TAG}/bin:${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}" +export PATH="/opt/python/${PYTHON_TAG}/bin:${PATH}" which python python --version which pip @@ -32,6 +31,7 @@ echo -e "\e[0Ksection_end:`date +%s`:setup_python\r\e[0K" # Install Rust echo -e "\e[0Ksection_start:`date +%s`:install_rust\r\e[0KInstalling Rust toolchain" +export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}" if ! command -v rustc &> /dev/null; then curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y fi From 10cc7400e3f4c8ccbc92c9455a7717615d2e1cd8 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 15:34:33 -0500 Subject: [PATCH 20/86] fix uv path? --- .gitlab/scripts/build-wheel-macos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/scripts/build-wheel-macos.sh b/.gitlab/scripts/build-wheel-macos.sh index 33eb6190466..c0817c39f96 100755 --- a/.gitlab/scripts/build-wheel-macos.sh +++ b/.gitlab/scripts/build-wheel-macos.sh @@ -20,7 +20,7 @@ echo -e "\e[0Ksection_end:`date +%s`:setup_env\r\e[0K" # Setup Python environment echo -e "\e[0Ksection_start:`date +%s`:setup_python\r\e[0KSetting up Python ${PYTHON_VERSION}" -export PATH="${HOME}/.uv/bin/:${PATH}" +export PATH="${UV_INSTALL_DIR}:${PATH}" if ! command -v uv &> /dev/null; then curl -LsSf https://astral.sh/uv/install.sh | sh fi From 45b854a27ee05aa6c57ad72c6448ddf78a901ca6 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 15:37:29 -0500 Subject: [PATCH 21/86] also include home local bin --- .gitlab/scripts/build-wheel-macos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/scripts/build-wheel-macos.sh b/.gitlab/scripts/build-wheel-macos.sh index c0817c39f96..848118525f5 100755 --- a/.gitlab/scripts/build-wheel-macos.sh +++ b/.gitlab/scripts/build-wheel-macos.sh @@ -20,7 +20,7 @@ echo -e "\e[0Ksection_end:`date +%s`:setup_env\r\e[0K" # Setup Python environment echo -e "\e[0Ksection_start:`date +%s`:setup_python\r\e[0KSetting up Python ${PYTHON_VERSION}" -export PATH="${UV_INSTALL_DIR}:${PATH}" +export PATH="${UV_INSTALL_DIR}:${HOME}/.local/bin:${PATH}" if ! command -v uv &> /dev/null; then curl -LsSf https://astral.sh/uv/install.sh | sh fi From b626b67bb8f17074490c3e3f2dba08d76a6f6457 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 15:47:24 -0500 Subject: [PATCH 22/86] fix downstream job depend --- .gitlab/benchmarks/macrobenchmarks.yml | 9 ++------- .gitlab/benchmarks/microbenchmarks.yml | 9 ++------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/.gitlab/benchmarks/macrobenchmarks.yml b/.gitlab/benchmarks/macrobenchmarks.yml index e5b99c810fa..a2c240f2213 100644 --- a/.gitlab/benchmarks/macrobenchmarks.yml +++ b/.gitlab/benchmarks/macrobenchmarks.yml @@ -23,13 +23,8 @@ candidate: - interruptible: true needs: - pipeline: $PARENT_PIPELINE_ID - job: build linux - parallel: - matrix: - - IMAGE_ARCH: "x86_64" - ARCH_TAG: "amd64" - PYTHON_TAG: "cp39-cp39" - LINUX_OS_IMAGE: "manylinux2014" + job: "build linux: [x86_64, amd64, cp39-cp39, manylinux2014]" + artifacts: true script: | cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./ echo "CANDIDATE_WHL=$(ls *.whl | head -n 1)" | tee candidate.env diff --git a/.gitlab/benchmarks/microbenchmarks.yml b/.gitlab/benchmarks/microbenchmarks.yml index 1561af40fe3..b8dc3b045c6 100644 --- a/.gitlab/benchmarks/microbenchmarks.yml +++ b/.gitlab/benchmarks/microbenchmarks.yml @@ -138,13 +138,8 @@ candidate: tags: [ "arch:amd64" ] needs: - pipeline: $PARENT_PIPELINE_ID - job: build linux - parallel: - matrix: - - IMAGE_ARCH: "x86_64" - ARCH_TAG: "amd64" - PYTHON_TAG: "cp39-cp39" - LINUX_OS_IMAGE: "manylinux2014" + job: "build linux: [x86_64, amd64, cp39-cp39, manylinux2014]" + artifacts: true script: | cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./ echo "CANDIDATE_WHL=$(ls *.whl | head -n 1)" | tee candidate.env From 5bc691ecf40f6eb78b4bbcec49fe741e5912893b Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 15:57:14 -0500 Subject: [PATCH 23/86] update macos build script to go all in on uv --- .gitlab/package.yml | 4 ++-- .gitlab/scripts/build-wheel-macos.sh | 19 ++++++------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index bffa49aa0c8..5c77edeaefb 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -77,10 +77,10 @@ variables: matrix: - IMAGE_ARCH: "x86_64" ARCH_TAG: "amd64" - PYTHON_VERSION: *PYTHON_VERSIONS + UV_PYTHON: *PYTHON_VERSIONS - IMAGE_ARCH: "aarch64" ARCH_TAG: "arm64" - PYTHON_VERSION: *PYTHON_VERSIONS + UV_PYTHON: *PYTHON_VERSIONS variables: SYSTEM_VERSION_COMPAT: "0" script: diff --git a/.gitlab/scripts/build-wheel-macos.sh b/.gitlab/scripts/build-wheel-macos.sh index 848118525f5..5cb385bed59 100755 --- a/.gitlab/scripts/build-wheel-macos.sh +++ b/.gitlab/scripts/build-wheel-macos.sh @@ -19,17 +19,11 @@ mkdir -p "${DEBUG_WHEEL_DIR}" echo -e "\e[0Ksection_end:`date +%s`:setup_env\r\e[0K" # Setup Python environment -echo -e "\e[0Ksection_start:`date +%s`:setup_python\r\e[0KSetting up Python ${PYTHON_VERSION}" +echo -e "\e[0Ksection_start:`date +%s`:setup_python\r\e[0KSetting up Python ${UV_PYTHON}" export PATH="${UV_INSTALL_DIR}:${HOME}/.local/bin:${PATH}" if ! command -v uv &> /dev/null; then curl -LsSf https://astral.sh/uv/install.sh | sh fi -uv python install "${PYTHON_VERSION}" -which python -python --version -which pip -pip --version -pip cache info echo -e "\e[0Ksection_end:`date +%s`:setup_python\r\e[0K" echo -e "\e[0Ksection_start:`date +%s`:install_rust\r\e[0KInstalling Rust toolchain" @@ -53,19 +47,19 @@ echo -e "\e[0Ksection_end:`date +%s`:install_sccache\r\e[0K" # Build wheel echo -e "\e[0Ksection_start:`date +%s`:build_wheel\r\e[0KBuilding wheel" -python -m build --wheel --outdir "${BUILT_WHEEL_DIR}" . +uv build --wheel --outdir "${BUILT_WHEEL_DIR}" . BUILT_WHEEL_FILE=$(ls ${BUILT_WHEEL_DIR}/*.whl | head -n 1) echo -e "\e[0Ksection_end:`date +%s`:build_wheel\r\e[0K" # Extract debug symbols echo -e "\e[0Ksection_start:`date +%s`:extract_debug_symbols\r\e[0KExtracting debug symbols" -python scripts/extract_debug_symbols.py "${BUILT_WHEEL_FILE}" --output-dir "${DEBUG_WHEEL_DIR}" +uv run scripts/extract_debug_symbols.py "${BUILT_WHEEL_FILE}" --output-dir "${DEBUG_WHEEL_DIR}" echo -e "\e[0Ksection_end:`date +%s`:extract_debug_symbols\r\e[0K" # Strip unneeded files from wheel echo -e "\e[0Ksection_start:`date +%s`:strip_wheel\r\e[0KStripping unneeded files from wheel" -python scripts/zip_filter.py "${BUILT_WHEEL_FILE}" \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md +uv run scripts/zip_filter.py "${BUILT_WHEEL_FILE}" \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md echo -e "\e[0Ksection_end:`date +%s`:strip_wheel\r\e[0K" # List all .so files in the wheel for verification @@ -75,7 +69,7 @@ echo -e "\e[0Ksection_end:`date +%s`:list_so_files\r\e[0K" # Repair the wheel echo -e "\e[0Ksection_start:`date +%s`:repair_wheel\r\e[0KRepairing wheel with delocate-wheel" -MACOSX_DEPLOYMENT_TARGET=12.7 delocate-wheel --require-archs "${ARCH_TAG}" -w "${TMP_WHEEL_DIR}" -v "${BUILT_WHEEL_FILE}" +MACOSX_DEPLOYMENT_TARGET=12.7 uvx --with="delocate" delocate-wheel --require-archs "${ARCH_TAG}" -w "${TMP_WHEEL_DIR}" -v "${BUILT_WHEEL_FILE}" echo -e "\e[0Ksection_end:`date +%s`:repair_wheel\r\e[0K" # Move to final resting place @@ -89,8 +83,7 @@ echo -e "\e[0Ksection_end:`date +%s`:finalize_wheel\r\e[0K" echo -e "\e[0Ksection_start:`date +%s`:test_wheel\r\e[0KTesting the wheel" TEST_WHEEL_DIR="/tmp/test_wheel/" mkdir -p "${TEST_WHEEL_DIR}" -python -m pip install virtualenv -python -m virtualenv --no-periodic-update --pip=embed --no-setuptools "${TEST_WHEEL_DIR}/venv" +uv venv "${TEST_WHEEL_DIR}/venv" # Install the package "${TEST_WHEEL_DIR}/venv/bin/pip" install "${FINAL_WHEEL_FILE}" # Run the smoke tests From aeaa13cdda172220ca61f845369a7840bd91e2a2 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 16:03:05 -0500 Subject: [PATCH 24/86] fix out dir for macos --- .gitlab/scripts/build-wheel-macos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/scripts/build-wheel-macos.sh b/.gitlab/scripts/build-wheel-macos.sh index 5cb385bed59..75c80bc5f36 100755 --- a/.gitlab/scripts/build-wheel-macos.sh +++ b/.gitlab/scripts/build-wheel-macos.sh @@ -47,7 +47,7 @@ echo -e "\e[0Ksection_end:`date +%s`:install_sccache\r\e[0K" # Build wheel echo -e "\e[0Ksection_start:`date +%s`:build_wheel\r\e[0KBuilding wheel" -uv build --wheel --outdir "${BUILT_WHEEL_DIR}" . +uv build --wheel --out-dir "${BUILT_WHEEL_DIR}" . BUILT_WHEEL_FILE=$(ls ${BUILT_WHEEL_DIR}/*.whl | head -n 1) echo -e "\e[0Ksection_end:`date +%s`:build_wheel\r\e[0K" From 3c135f0987ef9d9e122b59526e8d7ab0047793cf Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 19:46:10 -0500 Subject: [PATCH 25/86] from delocate --- .gitlab/scripts/build-wheel-macos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/scripts/build-wheel-macos.sh b/.gitlab/scripts/build-wheel-macos.sh index 75c80bc5f36..5c20e6715fb 100755 --- a/.gitlab/scripts/build-wheel-macos.sh +++ b/.gitlab/scripts/build-wheel-macos.sh @@ -69,7 +69,7 @@ echo -e "\e[0Ksection_end:`date +%s`:list_so_files\r\e[0K" # Repair the wheel echo -e "\e[0Ksection_start:`date +%s`:repair_wheel\r\e[0KRepairing wheel with delocate-wheel" -MACOSX_DEPLOYMENT_TARGET=12.7 uvx --with="delocate" delocate-wheel --require-archs "${ARCH_TAG}" -w "${TMP_WHEEL_DIR}" -v "${BUILT_WHEEL_FILE}" +MACOSX_DEPLOYMENT_TARGET=12.7 uvx --from="delocate" delocate-wheel --require-archs "${ARCH_TAG}" -w "${TMP_WHEEL_DIR}" -v "${BUILT_WHEEL_FILE}" echo -e "\e[0Ksection_end:`date +%s`:repair_wheel\r\e[0K" # Move to final resting place From a13709ef830a7926908b7d2e8c5f7a102e940314 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 19:47:12 -0500 Subject: [PATCH 26/86] public s3 depend on build macos --- .gitlab/package.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 5c77edeaefb..39dbc50ced3 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -179,6 +179,16 @@ publish-wheels-to-s3: ARCH_TAG: "arm64" PYTHON_TAG: *PYTHON_TAGS LINUX_OS_IMAGE: *LINUX_OS_IMAGES + - job: "build macos" + artifacts: true + parallel: + matrix: + - IMAGE_ARCH: "x86_64" + ARCH_TAG: "amd64" + UV_PYTHON: *PYTHON_VERSIONS + - IMAGE_ARCH: "aarch64" + ARCH_TAG: "arm64" + UV_PYTHON: *PYTHON_VERSIONS variables: BUCKET: dd-trace-py-builds script: From ea56c713c22a5380025040ab8a7a9fd906b70004 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 20:04:16 -0500 Subject: [PATCH 27/86] prefill with github variables --- .gitlab-ci.yml | 53 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1698d742131..1fab66bd5e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,6 +41,35 @@ include: - local: ".gitlab/benchmarks/serverless.yml" - local: ".gitlab/native.yml" +github variables: + image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 + tags: [ "arch:amd64" ] + stage: .pre + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + script: + - | + if [ -z ${GH_TOKEN} ] + then + # Use dd-octo-sts to get GitHub token + dd-octo-sts token --scope DataDog/dd-trace-py --policy gitlab.github-access.read > token + gh auth login --with-token < token + rm token + fi + - | + # Prevent git operation errors: + # failed to determine base repo: failed to run git: fatal: detected dubious ownership in repository at ... + git config --global --add safe.directory "${CI_PROJECT_DIR}" + - | + # Determine if we have an open GitHub PR for this commit + GH_PR_NUMBER=$(gh pr list --search "${CI_COMMIT_REF_NAME}"--json number --jq '.[0].number' || echo "") + echo "GH_PR_NUMBER=${GH_PR_NUMBER}" >> workflow.env + echo "GH_PR_OPEN=$(if [ -z "${GH_PR_NUMBER}" ]; then echo "false"; else echo "true"; fi)" >> workflow.env + artifacts: + reports: + dotenv: workflow.env + tests-gen: stage: tests extends: .testrunner @@ -101,9 +130,20 @@ microbenchmarks: PYTHON_TAG: "cp39-cp39" LINUX_OS_IMAGE: "manylinux2014" rules: + # Allow failures if explicitly asked - if: $RELEASE_ALLOW_BENCHMARK_FAILURES == "true" allow_failure: true - - allow_failure: false + # Always run on PRs, tagged releases, and main or release branches + - if: $GH_PR_OPEN == "true" + when: always + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+/ + when: always + - if: $CI_COMMIT_BRANCH == "main" + when: always + - if: $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+$/ + when: always + # Otherwise, run manually + - when: manual trigger: include: .gitlab/benchmarks/microbenchmarks.yml strategy: depend @@ -130,10 +170,17 @@ macrobenchmarks: DD_DISABLE_VPA: true allow_failure: true rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - when: always + # Allow failures if explicitly asked + - if: $RELEASE_ALLOW_BENCHMARK_FAILURES == "true" + allow_failure: true + # Always run on tagged releases, and main or release branches - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+/ when: always + - if: $CI_COMMIT_BRANCH == "main" + when: always + - if: $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+$/ + when: always + # Otherwise, run manually - when: manual check_new_flaky_tests: From d49c35467c16c306646e13b12042f51a28a22fdb Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 20:04:48 -0500 Subject: [PATCH 28/86] target macos version of 14.0 --- .gitlab/scripts/build-wheel-macos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/scripts/build-wheel-macos.sh b/.gitlab/scripts/build-wheel-macos.sh index 5c20e6715fb..fb7364bb1b9 100755 --- a/.gitlab/scripts/build-wheel-macos.sh +++ b/.gitlab/scripts/build-wheel-macos.sh @@ -69,7 +69,7 @@ echo -e "\e[0Ksection_end:`date +%s`:list_so_files\r\e[0K" # Repair the wheel echo -e "\e[0Ksection_start:`date +%s`:repair_wheel\r\e[0KRepairing wheel with delocate-wheel" -MACOSX_DEPLOYMENT_TARGET=12.7 uvx --from="delocate" delocate-wheel --require-archs "${ARCH_TAG}" -w "${TMP_WHEEL_DIR}" -v "${BUILT_WHEEL_FILE}" +MACOSX_DEPLOYMENT_TARGET=14.0 uvx --from="delocate" delocate-wheel --require-archs "${ARCH_TAG}" -w "${TMP_WHEEL_DIR}" -v "${BUILT_WHEEL_FILE}" echo -e "\e[0Ksection_end:`date +%s`:repair_wheel\r\e[0K" # Move to final resting place From de9034f0b83add2f41490db4e476bd748a7c2261 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 20:08:50 -0500 Subject: [PATCH 29/86] does the stage matter? --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1fab66bd5e3..1a8b8495bc8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,5 @@ stages: + - setup - package - tests - shared-pipeline @@ -44,7 +45,7 @@ include: github variables: image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 tags: [ "arch:amd64" ] - stage: .pre + stage: setup id_tokens: DDOCTOSTS_ID_TOKEN: aud: dd-octo-sts From 40d910b7a9ee7f146d68c89f37b9dd2e93d817b7 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 20:10:12 -0500 Subject: [PATCH 30/86] fix command --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1a8b8495bc8..26f9cd56363 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -64,7 +64,7 @@ github variables: git config --global --add safe.directory "${CI_PROJECT_DIR}" - | # Determine if we have an open GitHub PR for this commit - GH_PR_NUMBER=$(gh pr list --search "${CI_COMMIT_REF_NAME}"--json number --jq '.[0].number' || echo "") + GH_PR_NUMBER=$(gh pr list --search "${CI_COMMIT_REF_NAME}" --json number --jq '.[0].number' || echo "") echo "GH_PR_NUMBER=${GH_PR_NUMBER}" >> workflow.env echo "GH_PR_OPEN=$(if [ -z "${GH_PR_NUMBER}" ]; then echo "false"; else echo "true"; fi)" >> workflow.env artifacts: From 8188062c6209e8956337bdb6086c9ea730ad2a35 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 1 Dec 2025 20:13:04 -0500 Subject: [PATCH 31/86] don't clone --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 26f9cd56363..2065530bbc8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,6 +49,8 @@ github variables: id_tokens: DDOCTOSTS_ID_TOKEN: aud: dd-octo-sts + variables: + GIT_STRATEGY: none script: - | if [ -z ${GH_TOKEN} ] From 87d5317314a1cfa0a985ded21adbe24153f884be Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 2 Dec 2025 10:22:58 -0500 Subject: [PATCH 32/86] fix pipeline --- .gitlab-ci.yml | 2 +- .gitlab/scripts/build-wheel-macos.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2065530bbc8..447b82899bd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,7 +42,7 @@ include: - local: ".gitlab/benchmarks/serverless.yml" - local: ".gitlab/native.yml" -github variables: +pipeline variables: image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 tags: [ "arch:amd64" ] stage: setup diff --git a/.gitlab/scripts/build-wheel-macos.sh b/.gitlab/scripts/build-wheel-macos.sh index fb7364bb1b9..54e3742e6e3 100755 --- a/.gitlab/scripts/build-wheel-macos.sh +++ b/.gitlab/scripts/build-wheel-macos.sh @@ -69,7 +69,7 @@ echo -e "\e[0Ksection_end:`date +%s`:list_so_files\r\e[0K" # Repair the wheel echo -e "\e[0Ksection_start:`date +%s`:repair_wheel\r\e[0KRepairing wheel with delocate-wheel" -MACOSX_DEPLOYMENT_TARGET=14.0 uvx --from="delocate" delocate-wheel --require-archs "${ARCH_TAG}" -w "${TMP_WHEEL_DIR}" -v "${BUILT_WHEEL_FILE}" +MACOSX_DEPLOYMENT_TARGET=14.7 uvx --from="delocate" delocate-wheel --require-archs "${ARCH_TAG}" -w "${TMP_WHEEL_DIR}" -v "${BUILT_WHEEL_FILE}" echo -e "\e[0Ksection_end:`date +%s`:repair_wheel\r\e[0K" # Move to final resting place From 0d11e7d9598b85d0067b4c1d3bd8b736c68e9267 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 2 Dec 2025 10:31:20 -0500 Subject: [PATCH 33/86] refine pipeline variables --- .gitlab-ci.yml | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 447b82899bd..1acc37c72d1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -67,8 +67,11 @@ pipeline variables: - | # Determine if we have an open GitHub PR for this commit GH_PR_NUMBER=$(gh pr list --search "${CI_COMMIT_REF_NAME}" --json number --jq '.[0].number' || echo "") - echo "GH_PR_NUMBER=${GH_PR_NUMBER}" >> workflow.env - echo "GH_PR_OPEN=$(if [ -z "${GH_PR_NUMBER}" ]; then echo "false"; else echo "true"; fi)" >> workflow.env + echo "GH_PR_NUMBER=${GH_PR_NUMBER}" | tee -a workflow.env + echo "HAS_OPEN_PR=$(if [ -z "${GH_PR_NUMBER}" ]; then echo "false"; else echo "true"; fi)" | tee -a workflow.env + echo "IS_MAIN_BRANCH=$(if [ "${CI_COMMIT_BRANCH}" == "main" ]; then echo "true"; else echo "false"; fi)" | tee -a workflow.env + echo "IS_RELEASE_BRANCH=$(if [[ "${CI_COMMIT_BRANCH}" =~ ^[0-9]+\.[0-9]+$ ]]; then echo "true"; else echo "false"; fi)" | tee -a workflow.env + echo "IS_RELEASE=TAG=$(if [[ "${CI_COMMIT_TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo "true"; else echo "false"; fi)" | tee -a workflow.env artifacts: reports: dotenv: workflow.env @@ -137,13 +140,7 @@ microbenchmarks: - if: $RELEASE_ALLOW_BENCHMARK_FAILURES == "true" allow_failure: true # Always run on PRs, tagged releases, and main or release branches - - if: $GH_PR_OPEN == "true" - when: always - - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+/ - when: always - - if: $CI_COMMIT_BRANCH == "main" - when: always - - if: $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+$/ + - if: ($HAS_OPEN_PR == "true" || $IS_RELEASE == "true" || $IS_MAIN_BRANCH == "true" || $IS_RELEASE_BRANCH == "true") when: always # Otherwise, run manually - when: manual @@ -177,11 +174,7 @@ macrobenchmarks: - if: $RELEASE_ALLOW_BENCHMARK_FAILURES == "true" allow_failure: true # Always run on tagged releases, and main or release branches - - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+/ - when: always - - if: $CI_COMMIT_BRANCH == "main" - when: always - - if: $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+$/ + - if: ($IS_RELEASE == "true" || $IS_MAIN_BRANCH == "true" || $IS_RELEASE_BRANCH == "true") when: always # Otherwise, run manually - when: manual From e529b298c9061735f2ee92b96e1a062cdd39befb Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 2 Dec 2025 10:40:48 -0500 Subject: [PATCH 34/86] fix macos build? --- .gitlab/package.yml | 2 -- .gitlab/scripts/build-wheel-macos.sh | 7 ++++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 39dbc50ced3..31f5fa23335 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -120,8 +120,6 @@ download_dependency_wheels: tags: [ "arch:amd64" ] stage: package needs: - - job: download_ddtrace_artifacts - artifacts: true - job: "build linux" artifacts: true parallel: diff --git a/.gitlab/scripts/build-wheel-macos.sh b/.gitlab/scripts/build-wheel-macos.sh index 54e3742e6e3..bf87879ef63 100755 --- a/.gitlab/scripts/build-wheel-macos.sh +++ b/.gitlab/scripts/build-wheel-macos.sh @@ -54,12 +54,12 @@ echo -e "\e[0Ksection_end:`date +%s`:build_wheel\r\e[0K" # Extract debug symbols echo -e "\e[0Ksection_start:`date +%s`:extract_debug_symbols\r\e[0KExtracting debug symbols" -uv run scripts/extract_debug_symbols.py "${BUILT_WHEEL_FILE}" --output-dir "${DEBUG_WHEEL_DIR}" +uv run --no-project scripts/extract_debug_symbols.py "${BUILT_WHEEL_FILE}" --output-dir "${DEBUG_WHEEL_DIR}" echo -e "\e[0Ksection_end:`date +%s`:extract_debug_symbols\r\e[0K" # Strip unneeded files from wheel echo -e "\e[0Ksection_start:`date +%s`:strip_wheel\r\e[0KStripping unneeded files from wheel" -uv run scripts/zip_filter.py "${BUILT_WHEEL_FILE}" \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md +uv run --no-project scripts/zip_filter.py "${BUILT_WHEEL_FILE}" \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md echo -e "\e[0Ksection_end:`date +%s`:strip_wheel\r\e[0K" # List all .so files in the wheel for verification @@ -84,8 +84,9 @@ echo -e "\e[0Ksection_start:`date +%s`:test_wheel\r\e[0KTesting the wheel" TEST_WHEEL_DIR="/tmp/test_wheel/" mkdir -p "${TEST_WHEEL_DIR}" uv venv "${TEST_WHEEL_DIR}/venv" +source "${TEST_WHEEL_DIR}/venv/bin/activate" # Install the package -"${TEST_WHEEL_DIR}/venv/bin/pip" install "${FINAL_WHEEL_FILE}" +uv pip install "${FINAL_WHEEL_FILE}" # Run the smoke tests cd "${TEST_WHEEL_DIR}" && "${TEST_WHEEL_DIR}/venv/bin/python" "${PROJECT_DIR}/tests/smoke_test.py" echo -e "\e[0Ksection_end:`date +%s`:test_wheel\r\e[0K" From 3993e3cd28ac4fd14337bb57fa29813080bd6525 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 2 Dec 2025 10:44:13 -0500 Subject: [PATCH 35/86] reorg --- .gitlab-ci.yml | 2 +- .gitlab/package.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1acc37c72d1..9c070e1921a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -71,7 +71,7 @@ pipeline variables: echo "HAS_OPEN_PR=$(if [ -z "${GH_PR_NUMBER}" ]; then echo "false"; else echo "true"; fi)" | tee -a workflow.env echo "IS_MAIN_BRANCH=$(if [ "${CI_COMMIT_BRANCH}" == "main" ]; then echo "true"; else echo "false"; fi)" | tee -a workflow.env echo "IS_RELEASE_BRANCH=$(if [[ "${CI_COMMIT_BRANCH}" =~ ^[0-9]+\.[0-9]+$ ]]; then echo "true"; else echo "false"; fi)" | tee -a workflow.env - echo "IS_RELEASE=TAG=$(if [[ "${CI_COMMIT_TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo "true"; else echo "false"; fi)" | tee -a workflow.env + echo "IS_RELEASE=$(if [[ "${CI_COMMIT_TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo "true"; else echo "false"; fi)" | tee -a workflow.env artifacts: reports: dotenv: workflow.env diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 31f5fa23335..67b35a61c8d 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -25,18 +25,18 @@ variables: .build_base: stage: package variables: - CARGO_HOME: "${CI_PROJECT_DIR}/.cache/cargo" CMAKE_BUILD_PARALLEL_LEVEL: "12" CARGO_BUILD_JOBS: "12" CMAKE_ARGS: "-DNATIVE_TESTING=OFF" # DEV: Hack to make ASM CMake able to find the Python3 library CIBW_BUILD: "1" - DD_USE_SCCACHE: "1" - DD_SETUP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/download_cache" # Caching + DD_SETUP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/download_cache" + CARGO_HOME: "${CI_PROJECT_DIR}/.cache/cargo" PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip" UV_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/uv" UV_INSTALL_DIR: "${CI_PROJECT_DIR}/.uv/" + DD_USE_SCCACHE: "1" SCCACHE_DIR: "${CI_PROJECT_DIR}/.cache/sccache" SCCACHE_CACHE_SIZE: "2G" # job resource requests From 10517264cf5fb9402f0887aa0c995ca64f704b37 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 2 Dec 2025 13:52:56 -0500 Subject: [PATCH 36/86] unify linux and macos script around uv --- .gitlab/package.yml | 8 +- .gitlab/scripts/build-wheel-macos.sh | 96 ------------------- .gitlab/scripts/build-wheel-manylinux.sh | 105 --------------------- .gitlab/scripts/build-wheel.sh | 112 +++++++++++++++++++++++ 4 files changed, 117 insertions(+), 204 deletions(-) delete mode 100755 .gitlab/scripts/build-wheel-macos.sh delete mode 100755 .gitlab/scripts/build-wheel-manylinux.sh create mode 100755 .gitlab/scripts/build-wheel.sh diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 67b35a61c8d..c8afb59f6f8 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -63,7 +63,9 @@ variables: PYTHON_TAG: *PYTHON_TAGS LINUX_OS_IMAGE: *LINUX_OS_IMAGES script: - - .gitlab/scripts/build-wheel-manylinux.sh + - manylinux-interpreters ensure "${PYTHON_TAG}" + - export UV_PYTHON="/opt/python/${PYTHON_TAG}/bin/python" + - .gitlab/scripts/build-wheel.sh cache: - key: v0-build-linux-cache-${ARCH_TAG}-${PYTHON_TAG}-${LINUX_OS_IMAGE} paths: @@ -84,9 +86,9 @@ variables: variables: SYSTEM_VERSION_COMPAT: "0" script: - - .gitlab/scripts/build-wheel-macos.sh + - .gitlab/scripts/build-wheel.sh cache: - - key: v0-build-macos-cache-${ARCH_TAG}-${PYTHON_TAG} + - key: v0-build-macos-cache-${ARCH_TAG}-${UV_PYTHON} paths: - .cache - .uv diff --git a/.gitlab/scripts/build-wheel-macos.sh b/.gitlab/scripts/build-wheel-macos.sh deleted file mode 100755 index bf87879ef63..00000000000 --- a/.gitlab/scripts/build-wheel-macos.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo -e "\e[0Ksection_start:`date +%s`:setup_env\r\e[0KSetup environment" -PROJECT_DIR="${CI_PROJECT_DIR:-}" -if [ -z "${PROJECT_DIR}" ]; then - # DEV: This scripts dir but up two levels - PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -fi - -BUILT_WHEEL_DIR="/tmp/cibuildwheel/built_wheel" -TMP_WHEEL_DIR="/tmp/cibuildwheel/tmp_wheel" -FINAL_WHEEL_DIR="${PROJECT_DIR}/pywheels" -DEBUG_WHEEL_DIR="${PROJECT_DIR}/debugwheelhouse" -mkdir -p "${BUILT_WHEEL_DIR}" -mkdir -p "${TMP_WHEEL_DIR}" -mkdir -p "${FINAL_WHEEL_DIR}" -mkdir -p "${DEBUG_WHEEL_DIR}" -echo -e "\e[0Ksection_end:`date +%s`:setup_env\r\e[0K" - -# Setup Python environment -echo -e "\e[0Ksection_start:`date +%s`:setup_python\r\e[0KSetting up Python ${UV_PYTHON}" -export PATH="${UV_INSTALL_DIR}:${HOME}/.local/bin:${PATH}" -if ! command -v uv &> /dev/null; then - curl -LsSf https://astral.sh/uv/install.sh | sh -fi -echo -e "\e[0Ksection_end:`date +%s`:setup_python\r\e[0K" - -echo -e "\e[0Ksection_start:`date +%s`:install_rust\r\e[0KInstalling Rust toolchain" -export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}" -rustup update stable -rustup default stable -rustup target add aarch64-apple-darwin -which rustc -rustc --version -echo -e "\e[0Ksection_end:`date +%s`:install_rust\r\e[0K" - -# Install sccache via cargo -echo -e "\e[0Ksection_start:`date +%s`:install_sccache\r\e[0KInstalling sccache" -if ! command -v sccache &> /dev/null; then - cargo install sccache -fi -which sccache -sccache --version -sccache --show-stats -echo -e "\e[0Ksection_end:`date +%s`:install_sccache\r\e[0K" - -# Build wheel -echo -e "\e[0Ksection_start:`date +%s`:build_wheel\r\e[0KBuilding wheel" -uv build --wheel --out-dir "${BUILT_WHEEL_DIR}" . -BUILT_WHEEL_FILE=$(ls ${BUILT_WHEEL_DIR}/*.whl | head -n 1) -echo -e "\e[0Ksection_end:`date +%s`:build_wheel\r\e[0K" - - -# Extract debug symbols -echo -e "\e[0Ksection_start:`date +%s`:extract_debug_symbols\r\e[0KExtracting debug symbols" -uv run --no-project scripts/extract_debug_symbols.py "${BUILT_WHEEL_FILE}" --output-dir "${DEBUG_WHEEL_DIR}" -echo -e "\e[0Ksection_end:`date +%s`:extract_debug_symbols\r\e[0K" - -# Strip unneeded files from wheel -echo -e "\e[0Ksection_start:`date +%s`:strip_wheel\r\e[0KStripping unneeded files from wheel" -uv run --no-project scripts/zip_filter.py "${BUILT_WHEEL_FILE}" \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md -echo -e "\e[0Ksection_end:`date +%s`:strip_wheel\r\e[0K" - -# List all .so files in the wheel for verification -echo -e "\e[0Ksection_start:`date +%s`:list_so_files\r\e[0KListing .so files in the wheel" -unzip -l "${BUILT_WHEEL_FILE}" | grep '\.so$' -echo -e "\e[0Ksection_end:`date +%s`:list_so_files\r\e[0K" - -# Repair the wheel -echo -e "\e[0Ksection_start:`date +%s`:repair_wheel\r\e[0KRepairing wheel with delocate-wheel" -MACOSX_DEPLOYMENT_TARGET=14.7 uvx --from="delocate" delocate-wheel --require-archs "${ARCH_TAG}" -w "${TMP_WHEEL_DIR}" -v "${BUILT_WHEEL_FILE}" -echo -e "\e[0Ksection_end:`date +%s`:repair_wheel\r\e[0K" - -# Move to final resting place -echo -e "\e[0Ksection_start:`date +%s`:finalize_wheel\r\e[0KFinalizing wheel" -TMP_WHEEL_FILE=$(ls ${TMP_WHEEL_DIR}/*.whl | head -n 1) -mv "${TMP_WHEEL_FILE}" "${FINAL_WHEEL_DIR}/" -FINAL_WHEEL_FILE=$(ls ${FINAL_WHEEL_DIR}/*.whl | head -n 1) -echo -e "\e[0Ksection_end:`date +%s`:finalize_wheel\r\e[0K" - -# Test the wheel -echo -e "\e[0Ksection_start:`date +%s`:test_wheel\r\e[0KTesting the wheel" -TEST_WHEEL_DIR="/tmp/test_wheel/" -mkdir -p "${TEST_WHEEL_DIR}" -uv venv "${TEST_WHEEL_DIR}/venv" -source "${TEST_WHEEL_DIR}/venv/bin/activate" -# Install the package -uv pip install "${FINAL_WHEEL_FILE}" -# Run the smoke tests -cd "${TEST_WHEEL_DIR}" && "${TEST_WHEEL_DIR}/venv/bin/python" "${PROJECT_DIR}/tests/smoke_test.py" -echo -e "\e[0Ksection_end:`date +%s`:test_wheel\r\e[0K" - -echo -e "\e[0Ksection_start:`date +%s`:teardown\r\e[0KTearing down" -sccache --show-stats -echo -e "\e[0Ksection_end:`date +%s`:teardown\r\e[0K" diff --git a/.gitlab/scripts/build-wheel-manylinux.sh b/.gitlab/scripts/build-wheel-manylinux.sh deleted file mode 100755 index 60c29a2d7d9..00000000000 --- a/.gitlab/scripts/build-wheel-manylinux.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo -e "\e[0Ksection_start:`date +%s`:setup_env\r\e[0KSetup environment" -PROJECT_DIR="${CI_PROJECT_DIR:-}" -if [ -z "${PROJECT_DIR}" ]; then - # DEV: This scripts dir but up two levels - PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -fi - -BUILT_WHEEL_DIR="/tmp/cibuildwheel/built_wheel" -TMP_WHEEL_DIR="/tmp/cibuildwheel/tmp_wheel" -FINAL_WHEEL_DIR="${PROJECT_DIR}/pywheels" -DEBUG_WHEEL_DIR="${PROJECT_DIR}/debugwheelhouse" -mkdir -p "${BUILT_WHEEL_DIR}" -mkdir -p "${TMP_WHEEL_DIR}" -mkdir -p "${FINAL_WHEEL_DIR}" -mkdir -p "${DEBUG_WHEEL_DIR}" -echo -e "\e[0Ksection_end:`date +%s`:setup_env\r\e[0K" - -# Setup Python environment -echo -e "\e[0Ksection_start:`date +%s`:setup_python\r\e[0KSetting up Python ${PYTHON_TAG}" -manylinux-interpreters ensure "${PYTHON_TAG}" -export PATH="/opt/python/${PYTHON_TAG}/bin:${PATH}" -which python -python --version -which pip -pip --version -pip cache info -echo -e "\e[0Ksection_end:`date +%s`:setup_python\r\e[0K" - -# Install Rust -echo -e "\e[0Ksection_start:`date +%s`:install_rust\r\e[0KInstalling Rust toolchain" -export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}" -if ! command -v rustc &> /dev/null; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -fi -rustup default stable -which rustc -rustc --version -echo -e "\e[0Ksection_end:`date +%s`:install_rust\r\e[0K" - -# Install sccache via cargo -echo -e "\e[0Ksection_start:`date +%s`:install_sccache\r\e[0KInstalling sccache" -if ! command -v sccache &> /dev/null; then - if command -v yum &> /dev/null; then - yum install -y openssl-devel - elif command -v apk &> /dev/null; then - apk --no-cache add openssl-dev openssl-libs-static - fi - cargo install sccache -fi -which sccache -sccache --version -sccache --show-stats -echo -e "\e[0Ksection_end:`date +%s`:install_sccache\r\e[0K" - -# Build wheel -echo -e "\e[0Ksection_start:`date +%s`:build_wheel\r\e[0KBuilding wheel" -python -m build --wheel --outdir "${BUILT_WHEEL_DIR}" . -BUILT_WHEEL_FILE=$(ls ${BUILT_WHEEL_DIR}/*.whl | head -n 1) -echo -e "\e[0Ksection_end:`date +%s`:build_wheel\r\e[0K" - -# Extract debug symbols -echo -e "\e[0Ksection_start:`date +%s`:extract_debug_symbols\r\e[0KExtracting debug symbols" -python scripts/extract_debug_symbols.py "${BUILT_WHEEL_FILE}" --output-dir "${DEBUG_WHEEL_DIR}" -echo -e "\e[0Ksection_end:`date +%s`:extract_debug_symbols\r\e[0K" - -# Strip unneeded files from wheel -echo -e "\e[0Ksection_start:`date +%s`:strip_wheel\r\e[0KStripping unneeded files from wheel" -python scripts/zip_filter.py "${BUILT_WHEEL_FILE}" \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md -echo -e "\e[0Ksection_end:`date +%s`:strip_wheel\r\e[0K" - -# List all .so files in the wheel for verification -echo -e "\e[0Ksection_start:`date +%s`:list_so_files\r\e[0KListing .so files in the wheel" -unzip -l "${BUILT_WHEEL_FILE}" | grep '\.so$' -echo -e "\e[0Ksection_end:`date +%s`:list_so_files\r\e[0K" - -# Repair the wheel -echo -e "\e[0Ksection_start:`date +%s`:repair_wheel\r\e[0KRepairing wheel with auditwheel" -auditwheel repair -w "${TMP_WHEEL_DIR}" "${BUILT_WHEEL_FILE}" -echo -e "\e[0Ksection_end:`date +%s`:repair_wheel\r\e[0K" - -# Move to final resting place -echo -e "\e[0Ksection_start:`date +%s`:finalize_wheel\r\e[0KFinalizing wheel" -TMP_WHEEL_FILE=$(ls ${TMP_WHEEL_DIR}/*.whl | head -n 1) -mv "${TMP_WHEEL_FILE}" "${FINAL_WHEEL_DIR}/" -FINAL_WHEEL_FILE=$(ls ${FINAL_WHEEL_DIR}/*.whl | head -n 1) -echo -e "\e[0Ksection_end:`date +%s`:finalize_wheel\r\e[0K" - -# Test the wheel -echo -e "\e[0Ksection_start:`date +%s`:test_wheel\r\e[0KTesting the wheel" -TEST_WHEEL_DIR="/tmp/test_wheel/" -mkdir -p "${TEST_WHEEL_DIR}" -python -m pip install virtualenv -python -m virtualenv --no-periodic-update --pip=embed --no-setuptools "${TEST_WHEEL_DIR}/venv" -# Install the package -"${TEST_WHEEL_DIR}/venv/bin/pip" install "${FINAL_WHEEL_FILE}" -# Run the smoke tests -cd "${TEST_WHEEL_DIR}" && "${TEST_WHEEL_DIR}/venv/bin/python" "${PROJECT_DIR}/tests/smoke_test.py" -echo -e "\e[0Ksection_end:`date +%s`:test_wheel\r\e[0K" - -echo -e "\e[0Ksection_start:`date +%s`:teardown\r\e[0KTearing down" -sccache --show-stats -echo -e "\e[0Ksection_end:`date +%s`:teardown\r\e[0K" diff --git a/.gitlab/scripts/build-wheel.sh b/.gitlab/scripts/build-wheel.sh new file mode 100755 index 00000000000..686577cb6d8 --- /dev/null +++ b/.gitlab/scripts/build-wheel.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Helper functions for GitLab CI collapsible sections +section_start() { + echo -e "\e[0Ksection_start:`date +%s`:$1\r\e[0K$2" +} + +section_end() { + echo -e "\e[0Ksection_end:`date +%s`:$1\r\e[0K" +} + +# Setup directories +section_start "setup_env" "Setup environment" +PROJECT_DIR="${CI_PROJECT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" +BUILT_WHEEL_DIR="/tmp/cibuildwheel/built_wheel" +TMP_WHEEL_DIR="/tmp/cibuildwheel/tmp_wheel" +FINAL_WHEEL_DIR="${PROJECT_DIR}/pywheels" +DEBUG_WHEEL_DIR="${PROJECT_DIR}/debugwheelhouse" +mkdir -p "${BUILT_WHEEL_DIR}" "${TMP_WHEEL_DIR}" "${FINAL_WHEEL_DIR}" "${DEBUG_WHEEL_DIR}" +section_end "setup_env" + +# Setup Python (verify/install uv if needed) +section_start "setup_python" "Setting up Python ${UV_PYTHON}" +# If UV_PYTHON is a full path (manylinux), add its bin directory to PATH +if [[ "${UV_PYTHON}" == /* ]]; then + export PATH="$(dirname "${UV_PYTHON}"):${PATH}" +fi +if ! command -v uv &> /dev/null; then + curl -LsSf https://astral.sh/uv/install.sh | sh + export PATH="${HOME}/.local/bin:${PATH}" +fi +which python && python --version +section_end "setup_python" + +# Setup Rust (verify/install if needed) +section_start "install_rust" "Rust toolchain" +export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}" +if ! command -v rustc &> /dev/null; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +fi +rustup default stable +which rustc && rustc --version +section_end "install_rust" + +# Setup sccache (verify/install if needed) +section_start "install_sccache" "sccache" +if ! command -v sccache &> /dev/null; then + # Install openssl-devel for building sccache + if command -v yum &> /dev/null; then + yum install -y openssl-devel + elif command -v apk &> /dev/null; then + apk --no-cache add openssl-dev openssl-libs-static + fi + cargo install sccache +fi +which sccache && sccache --version && sccache --show-stats +section_end "install_sccache" + +# Build wheel +section_start "build_wheel" "Building wheel" +uv build --wheel --out-dir "${BUILT_WHEEL_DIR}" . +BUILT_WHEEL_FILE=$(ls ${BUILT_WHEEL_DIR}/*.whl | head -n 1) +section_end "build_wheel" + +# Extract debug symbols +section_start "extract_debug_symbols" "Extracting debug symbols" +uv run --no-project scripts/extract_debug_symbols.py "${BUILT_WHEEL_FILE}" --output-dir "${DEBUG_WHEEL_DIR}" +section_end "extract_debug_symbols" + +# Strip wheel +section_start "strip_wheel" "Stripping unneeded files" +uv run --no-project scripts/zip_filter.py "${BUILT_WHEEL_FILE}" \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md +section_end "strip_wheel" + +# List .so files +section_start "list_so_files" "Listing .so files" +unzip -l "${BUILT_WHEEL_FILE}" | grep '\.so$' +section_end "list_so_files" + +# Repair wheel (ONLY PLATFORM-SPECIFIC CODE) +section_start "repair_wheel" "Repairing wheel" +if [[ "$(uname -s)" == "Linux" ]]; then + auditwheel repair -w "${TMP_WHEEL_DIR}" "${BUILT_WHEEL_FILE}" +else + # macOS + MACOSX_DEPLOYMENT_TARGET=14.7 uvx --from="delocate" delocate-wheel \ + --require-archs "${ARCH_TAG}" -w "${TMP_WHEEL_DIR}" -v "${BUILT_WHEEL_FILE}" +fi +section_end "repair_wheel" + +# Finalize +section_start "finalize_wheel" "Finalizing wheel" +TMP_WHEEL_FILE=$(ls ${TMP_WHEEL_DIR}/*.whl | head -n 1) +mv "${TMP_WHEEL_FILE}" "${FINAL_WHEEL_DIR}/" +FINAL_WHEEL_FILE=$(ls ${FINAL_WHEEL_DIR}/*.whl | head -n 1) +section_end "finalize_wheel" + +# Test wheel +section_start "test_wheel" "Testing wheel" +TEST_WHEEL_DIR="/tmp/test_wheel/" +mkdir -p "${TEST_WHEEL_DIR}" +uv venv "${TEST_WHEEL_DIR}/venv" +source "${TEST_WHEEL_DIR}/venv/bin/activate" +uv pip install "${FINAL_WHEEL_FILE}" +cd "${TEST_WHEEL_DIR}" && python "${PROJECT_DIR}/tests/smoke_test.py" +section_end "test_wheel" + +# Teardown +section_start "teardown" "Teardown" +sccache --show-stats +section_end "teardown" From ae09d252e9218bf5358e8a0d7ae658ef3370eff7 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 2 Dec 2025 14:49:54 -0500 Subject: [PATCH 37/86] fix path --- .gitlab/scripts/build-wheel.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/scripts/build-wheel.sh b/.gitlab/scripts/build-wheel.sh index 686577cb6d8..d0ca5e97205 100755 --- a/.gitlab/scripts/build-wheel.sh +++ b/.gitlab/scripts/build-wheel.sh @@ -22,13 +22,14 @@ section_end "setup_env" # Setup Python (verify/install uv if needed) section_start "setup_python" "Setting up Python ${UV_PYTHON}" +# Set up PATH for uv and system tools +export PATH="${UV_INSTALL_DIR:-$HOME/.local/bin}:${PATH}" # If UV_PYTHON is a full path (manylinux), add its bin directory to PATH if [[ "${UV_PYTHON}" == /* ]]; then export PATH="$(dirname "${UV_PYTHON}"):${PATH}" fi if ! command -v uv &> /dev/null; then curl -LsSf https://astral.sh/uv/install.sh | sh - export PATH="${HOME}/.local/bin:${PATH}" fi which python && python --version section_end "setup_python" From 57b3d092e57522ea48d8dd3f05ffad0a27335adf Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Thu, 4 Dec 2025 15:08:14 -0500 Subject: [PATCH 38/86] mktemp and reduce number of macos runners --- .gitlab/package.yml | 38 ++++++++++++++++++++++++++++------ .gitlab/scripts/build-wheel.sh | 6 ++++-- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index c8afb59f6f8..2b4f92fff26 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -79,16 +79,44 @@ variables: matrix: - IMAGE_ARCH: "x86_64" ARCH_TAG: "amd64" - UV_PYTHON: *PYTHON_VERSIONS - IMAGE_ARCH: "aarch64" ARCH_TAG: "arm64" - UV_PYTHON: *PYTHON_VERSIONS variables: SYSTEM_VERSION_COMPAT: "0" script: - - .gitlab/scripts/build-wheel.sh + - | + set -euo pipefail + PYTHON_VERSIONS=(3.9 3.10 3.11 3.12 3.13 3.14) + BUILD_PIDS=() + FAILED=0 + + for PYTHON_VERSION in "${PYTHON_VERSIONS[@]}"; do + ( + export UV_PYTHON="${PYTHON_VERSION}" + .gitlab/scripts/build-wheel.sh + ) & + BUILD_PIDS+=($!) + echo "Started build for Python ${PYTHON_VERSION} (PID: $!)" + done + + echo "Waiting for all builds to complete..." + for i in "${!BUILD_PIDS[@]}"; do + PID=${BUILD_PIDS[$i]} + PYTHON_VERSION=${PYTHON_VERSIONS[$i]} + if wait "$PID"; then + echo "✓ Python ${PYTHON_VERSION} build completed successfully" + else + echo "✗ Python ${PYTHON_VERSION} build failed" + FAILED=1 + fi + done + + if [ $FAILED -eq 1 ]; then + echo "One or more builds failed" + exit 1 + fi cache: - - key: v0-build-macos-cache-${ARCH_TAG}-${UV_PYTHON} + - key: v0-build-macos-cache-${ARCH_TAG} paths: - .cache - .uv @@ -185,10 +213,8 @@ publish-wheels-to-s3: matrix: - IMAGE_ARCH: "x86_64" ARCH_TAG: "amd64" - UV_PYTHON: *PYTHON_VERSIONS - IMAGE_ARCH: "aarch64" ARCH_TAG: "arm64" - UV_PYTHON: *PYTHON_VERSIONS variables: BUCKET: dd-trace-py-builds script: diff --git a/.gitlab/scripts/build-wheel.sh b/.gitlab/scripts/build-wheel.sh index d0ca5e97205..2643ce0c999 100755 --- a/.gitlab/scripts/build-wheel.sh +++ b/.gitlab/scripts/build-wheel.sh @@ -13,8 +13,10 @@ section_end() { # Setup directories section_start "setup_env" "Setup environment" PROJECT_DIR="${CI_PROJECT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" -BUILT_WHEEL_DIR="/tmp/cibuildwheel/built_wheel" -TMP_WHEEL_DIR="/tmp/cibuildwheel/tmp_wheel" +WORK_DIR=$(mktemp -d) +trap "rm -rf '${WORK_DIR}'" EXIT +BUILT_WHEEL_DIR="${WORK_DIR}/built_wheel" +TMP_WHEEL_DIR="${WORK_DIR}/tmp_wheel" FINAL_WHEEL_DIR="${PROJECT_DIR}/pywheels" DEBUG_WHEEL_DIR="${PROJECT_DIR}/debugwheelhouse" mkdir -p "${BUILT_WHEEL_DIR}" "${TMP_WHEEL_DIR}" "${FINAL_WHEEL_DIR}" "${DEBUG_WHEEL_DIR}" From 107d76accb28058aa4a875138fa52938f458e3b7 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Thu, 4 Dec 2025 15:11:16 -0500 Subject: [PATCH 39/86] fix uv venv command --- .gitlab/scripts/build-wheel.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/scripts/build-wheel.sh b/.gitlab/scripts/build-wheel.sh index 2643ce0c999..4a32b64dd58 100755 --- a/.gitlab/scripts/build-wheel.sh +++ b/.gitlab/scripts/build-wheel.sh @@ -101,9 +101,9 @@ section_end "finalize_wheel" # Test wheel section_start "test_wheel" "Testing wheel" -TEST_WHEEL_DIR="/tmp/test_wheel/" +TEST_WHEEL_DIR="${WORK_DIR}/test_wheel" mkdir -p "${TEST_WHEEL_DIR}" -uv venv "${TEST_WHEEL_DIR}/venv" +uv venv --python="${UV_PYTHON}" "${TEST_WHEEL_DIR}/venv" source "${TEST_WHEEL_DIR}/venv/bin/activate" uv pip install "${FINAL_WHEEL_FILE}" cd "${TEST_WHEEL_DIR}" && python "${PROJECT_DIR}/tests/smoke_test.py" From 27a654346054fe70d5d07744210f75814b09cabd Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Thu, 4 Dec 2025 15:56:55 -0500 Subject: [PATCH 40/86] fix linux, limit macos builds --- .gitlab/package.yml | 44 ++++++++++------------------------ .gitlab/scripts/build-wheel.sh | 9 ++++--- 2 files changed, 18 insertions(+), 35 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 2b4f92fff26..2bc4ef4e7e5 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -79,44 +79,22 @@ variables: matrix: - IMAGE_ARCH: "x86_64" ARCH_TAG: "amd64" + UV_PYTHON: *PYTHON_VERSIONS - IMAGE_ARCH: "aarch64" ARCH_TAG: "arm64" + UV_PYTHON: *PYTHON_VERSIONS variables: SYSTEM_VERSION_COMPAT: "0" + rules: + # Skip non-3.14 versions on PRs/feature branches - only run 3.14 for quick validation + - if: '($IS_MAIN_BRANCH == "false") && ($IS_RELEASE_BRANCH == "false") && ($IS_RELEASE == "false") && ($UV_PYTHON != "3.14")' + when: never + # Run all versions on main, release branches, and releases + - when: always script: - - | - set -euo pipefail - PYTHON_VERSIONS=(3.9 3.10 3.11 3.12 3.13 3.14) - BUILD_PIDS=() - FAILED=0 - - for PYTHON_VERSION in "${PYTHON_VERSIONS[@]}"; do - ( - export UV_PYTHON="${PYTHON_VERSION}" - .gitlab/scripts/build-wheel.sh - ) & - BUILD_PIDS+=($!) - echo "Started build for Python ${PYTHON_VERSION} (PID: $!)" - done - - echo "Waiting for all builds to complete..." - for i in "${!BUILD_PIDS[@]}"; do - PID=${BUILD_PIDS[$i]} - PYTHON_VERSION=${PYTHON_VERSIONS[$i]} - if wait "$PID"; then - echo "✓ Python ${PYTHON_VERSION} build completed successfully" - else - echo "✗ Python ${PYTHON_VERSION} build failed" - FAILED=1 - fi - done - - if [ $FAILED -eq 1 ]; then - echo "One or more builds failed" - exit 1 - fi + - .gitlab/scripts/build-wheel.sh cache: - - key: v0-build-macos-cache-${ARCH_TAG} + - key: v0-build-macos-cache-${ARCH_TAG}-${UV_PYTHON} paths: - .cache - .uv @@ -213,8 +191,10 @@ publish-wheels-to-s3: matrix: - IMAGE_ARCH: "x86_64" ARCH_TAG: "amd64" + UV_PYTHON: *PYTHON_VERSIONS - IMAGE_ARCH: "aarch64" ARCH_TAG: "arm64" + UV_PYTHON: *PYTHON_VERSIONS variables: BUCKET: dd-trace-py-builds script: diff --git a/.gitlab/scripts/build-wheel.sh b/.gitlab/scripts/build-wheel.sh index 4a32b64dd58..c7ca7c3c9af 100755 --- a/.gitlab/scripts/build-wheel.sh +++ b/.gitlab/scripts/build-wheel.sh @@ -103,10 +103,13 @@ section_end "finalize_wheel" section_start "test_wheel" "Testing wheel" TEST_WHEEL_DIR="${WORK_DIR}/test_wheel" mkdir -p "${TEST_WHEEL_DIR}" -uv venv --python="${UV_PYTHON}" "${TEST_WHEEL_DIR}/venv" -source "${TEST_WHEEL_DIR}/venv/bin/activate" +VENV_PATH="${TEST_WHEEL_DIR}/venv" +uv venv --python="${UV_PYTHON}" "${VENV_PATH}" +export VIRTUAL_ENV="${VENV_PATH}" +export PATH="${VENV_PATH}/bin:${PATH}" +cd "${TEST_WHEEL_DIR}" uv pip install "${FINAL_WHEEL_FILE}" -cd "${TEST_WHEEL_DIR}" && python "${PROJECT_DIR}/tests/smoke_test.py" +"${VENV_PATH}/bin/python" "${PROJECT_DIR}/tests/smoke_test.py" section_end "test_wheel" # Teardown From 20174d3a9670b092d68c271f36c7e915b3853d88 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Fri, 5 Dec 2025 15:42:50 -0500 Subject: [PATCH 41/86] fix macos matrix? --- .gitlab/package.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 2bc4ef4e7e5..271a7c62ae7 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -79,18 +79,18 @@ variables: matrix: - IMAGE_ARCH: "x86_64" ARCH_TAG: "amd64" - UV_PYTHON: *PYTHON_VERSIONS + UV_PYTHON: $MACOS_PYTHON_VERSIONS - IMAGE_ARCH: "aarch64" ARCH_TAG: "arm64" - UV_PYTHON: *PYTHON_VERSIONS + UV_PYTHON: $MACOS_PYTHON_VERSIONS variables: SYSTEM_VERSION_COMPAT: "0" + MACOS_PYTHON_VERSIONS: "3.14" # Default: single version for PRs/feature branches rules: - # Skip non-3.14 versions on PRs/feature branches - only run 3.14 for quick validation - - if: '($IS_MAIN_BRANCH == "false") && ($IS_RELEASE_BRANCH == "false") && ($IS_RELEASE == "false") && ($UV_PYTHON != "3.14")' - when: never - # Run all versions on main, release branches, and releases - - when: always + # Full matrix on main, release branches, and releases + - if: '($IS_MAIN_BRANCH == "true") || ($IS_RELEASE_BRANCH == "true") || ($IS_RELEASE == "true")' + variables: + MACOS_PYTHON_VERSIONS: "3.9 3.10 3.11 3.12 3.13 3.14" script: - .gitlab/scripts/build-wheel.sh cache: From 908e81f0816376379faa6b2a418c50654654f264 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Fri, 5 Dec 2025 15:52:18 -0500 Subject: [PATCH 42/86] debug --- .gitlab/scripts/build-wheel.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.gitlab/scripts/build-wheel.sh b/.gitlab/scripts/build-wheel.sh index c7ca7c3c9af..fa0e7115257 100755 --- a/.gitlab/scripts/build-wheel.sh +++ b/.gitlab/scripts/build-wheel.sh @@ -109,6 +109,22 @@ export VIRTUAL_ENV="${VENV_PATH}" export PATH="${VENV_PATH}/bin:${PATH}" cd "${TEST_WHEEL_DIR}" uv pip install "${FINAL_WHEEL_FILE}" + +# Diagnostics before running smoke test +echo "=== Environment Diagnostics ===" +echo "VIRTUAL_ENV: ${VIRTUAL_ENV}" +echo "PATH: ${PATH}" +echo "which python: $(which python)" +"${VENV_PATH}/bin/python" --version +echo "=== pip freeze ===" +"${VENV_PATH}/bin/python" -m pip freeze +echo "=== site-packages contents ===" +"${VENV_PATH}/bin/python" -c "import site; print('site-packages:', site.getsitepackages())" +ls -la "${VENV_PATH}/lib/"*/site-packages/ | head -30 +echo "=== Testing direct import ===" +"${VENV_PATH}/bin/python" -c "import ddtrace; print('✓ ddtrace import successful')" || echo "✗ ddtrace import failed" + +# Run smoke test "${VENV_PATH}/bin/python" "${PROJECT_DIR}/tests/smoke_test.py" section_end "test_wheel" From bfc36682b8e8fecd53fce1ae7d17758f551976a4 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Fri, 5 Dec 2025 15:56:29 -0500 Subject: [PATCH 43/86] fix publish to s3 needs --- .gitlab/package.yml | 75 ++++++++++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 271a7c62ae7..ccfd05619f5 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -170,31 +170,56 @@ publish-wheels-to-s3: tags: ["arch:amd64"] image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29 stage: package - needs: - # - job: download_ddtrace_artifacts - # artifacts: true - - job: "build linux" - artifacts: true - parallel: - matrix: - - IMAGE_ARCH: "x86_64" - ARCH_TAG: "amd64" - PYTHON_TAG: *PYTHON_TAGS - LINUX_OS_IMAGE: *LINUX_OS_IMAGES - - IMAGE_ARCH: "aarch64" - ARCH_TAG: "arm64" - PYTHON_TAG: *PYTHON_TAGS - LINUX_OS_IMAGE: *LINUX_OS_IMAGES - - job: "build macos" - artifacts: true - parallel: - matrix: - - IMAGE_ARCH: "x86_64" - ARCH_TAG: "amd64" - UV_PYTHON: *PYTHON_VERSIONS - - IMAGE_ARCH: "aarch64" - ARCH_TAG: "arm64" - UV_PYTHON: *PYTHON_VERSIONS + rules: + # On PRs/feature branches: only depend on 3.14 macos builds + - if: '($IS_MAIN_BRANCH == "false") && ($IS_RELEASE_BRANCH == "false") && ($IS_RELEASE == "false")' + needs: + - job: "build linux" + artifacts: true + parallel: + matrix: + - IMAGE_ARCH: "x86_64" + ARCH_TAG: "amd64" + PYTHON_TAG: *PYTHON_TAGS + LINUX_OS_IMAGE: *LINUX_OS_IMAGES + - IMAGE_ARCH: "aarch64" + ARCH_TAG: "arm64" + PYTHON_TAG: *PYTHON_TAGS + LINUX_OS_IMAGE: *LINUX_OS_IMAGES + - job: "build macos" + artifacts: true + parallel: + matrix: + - IMAGE_ARCH: "x86_64" + ARCH_TAG: "amd64" + UV_PYTHON: "3.14" + - IMAGE_ARCH: "aarch64" + ARCH_TAG: "arm64" + UV_PYTHON: "3.14" + # On main, release branches, and releases: depend on all macos builds + - needs: + - job: "build linux" + artifacts: true + parallel: + matrix: + - IMAGE_ARCH: "x86_64" + ARCH_TAG: "amd64" + PYTHON_TAG: *PYTHON_TAGS + LINUX_OS_IMAGE: *LINUX_OS_IMAGES + - IMAGE_ARCH: "aarch64" + ARCH_TAG: "arm64" + PYTHON_TAG: *PYTHON_TAGS + LINUX_OS_IMAGE: *LINUX_OS_IMAGES + - job: "build macos" + artifacts: true + parallel: + matrix: + - IMAGE_ARCH: "x86_64" + ARCH_TAG: "amd64" + UV_PYTHON: *PYTHON_VERSIONS + - IMAGE_ARCH: "aarch64" + ARCH_TAG: "arm64" + UV_PYTHON: *PYTHON_VERSIONS variables: BUCKET: dd-trace-py-builds script: From 1e7507823bddc3cfa3f86b21d1ff288ad61b8fb5 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Fri, 5 Dec 2025 15:59:41 -0500 Subject: [PATCH 44/86] simplify? --- .gitlab/package.yml | 55 +++++---------------------------------------- 1 file changed, 5 insertions(+), 50 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index ccfd05619f5..2d5059adaee 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -170,56 +170,11 @@ publish-wheels-to-s3: tags: ["arch:amd64"] image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29 stage: package - rules: - # On PRs/feature branches: only depend on 3.14 macos builds - - if: '($IS_MAIN_BRANCH == "false") && ($IS_RELEASE_BRANCH == "false") && ($IS_RELEASE == "false")' - needs: - - job: "build linux" - artifacts: true - parallel: - matrix: - - IMAGE_ARCH: "x86_64" - ARCH_TAG: "amd64" - PYTHON_TAG: *PYTHON_TAGS - LINUX_OS_IMAGE: *LINUX_OS_IMAGES - - IMAGE_ARCH: "aarch64" - ARCH_TAG: "arm64" - PYTHON_TAG: *PYTHON_TAGS - LINUX_OS_IMAGE: *LINUX_OS_IMAGES - - job: "build macos" - artifacts: true - parallel: - matrix: - - IMAGE_ARCH: "x86_64" - ARCH_TAG: "amd64" - UV_PYTHON: "3.14" - - IMAGE_ARCH: "aarch64" - ARCH_TAG: "arm64" - UV_PYTHON: "3.14" - # On main, release branches, and releases: depend on all macos builds - - needs: - - job: "build linux" - artifacts: true - parallel: - matrix: - - IMAGE_ARCH: "x86_64" - ARCH_TAG: "amd64" - PYTHON_TAG: *PYTHON_TAGS - LINUX_OS_IMAGE: *LINUX_OS_IMAGES - - IMAGE_ARCH: "aarch64" - ARCH_TAG: "arm64" - PYTHON_TAG: *PYTHON_TAGS - LINUX_OS_IMAGE: *LINUX_OS_IMAGES - - job: "build macos" - artifacts: true - parallel: - matrix: - - IMAGE_ARCH: "x86_64" - ARCH_TAG: "amd64" - UV_PYTHON: *PYTHON_VERSIONS - - IMAGE_ARCH: "aarch64" - ARCH_TAG: "arm64" - UV_PYTHON: *PYTHON_VERSIONS + needs: + - job: "build linux" + artifacts: true + - job: "build macos" + artifacts: true variables: BUCKET: dd-trace-py-builds script: From 785596f2bc4914c30727e555f2f2bdca85a93203 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Fri, 5 Dec 2025 16:01:30 -0500 Subject: [PATCH 45/86] how about now? --- .gitlab/package.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 2d5059adaee..99a45cdc439 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -79,18 +79,16 @@ variables: matrix: - IMAGE_ARCH: "x86_64" ARCH_TAG: "amd64" - UV_PYTHON: $MACOS_PYTHON_VERSIONS + UV_PYTHON: *PYTHON_VERSIONS - IMAGE_ARCH: "aarch64" ARCH_TAG: "arm64" - UV_PYTHON: $MACOS_PYTHON_VERSIONS + UV_PYTHON: *PYTHON_VERSIONS variables: SYSTEM_VERSION_COMPAT: "0" - MACOS_PYTHON_VERSIONS: "3.14" # Default: single version for PRs/feature branches rules: - # Full matrix on main, release branches, and releases - - if: '($IS_MAIN_BRANCH == "true") || ($IS_RELEASE_BRANCH == "true") || ($IS_RELEASE == "true")' - variables: - MACOS_PYTHON_VERSIONS: "3.9 3.10 3.11 3.12 3.13 3.14" + # Skip non-3.14 versions on PRs/feature branches + - if: '($IS_MAIN_BRANCH == "false") && ($IS_RELEASE_BRANCH == "false") && ($IS_RELEASE == "false") && ($UV_PYTHON != "3.14")' + when: never script: - .gitlab/scripts/build-wheel.sh cache: From 8ea4d38f7dd46fcc4e3c5c26d3a3300de256c79a Mon Sep 17 00:00:00 2001 From: Vlad Scherbich Date: Fri, 5 Dec 2025 18:06:05 -0500 Subject: [PATCH 46/86] chore(profiling): fix 'test_semaphore_and_bounded_semaphore_collectors_coexist' test (#15536) ## Description Fix a segfault in the test_semaphore_and_bounded_semaphore_collectors_coexist test that occurred on Python 3.12. The test was missing the required ddup.config() and ddup.start() initialization before using lock collectors. This test was introduced in #15375 (unsure how it passed there...), but it should be working now. ## What Happened The test used `ThreadingSemaphoreCollector` and `ThreadingBoundedSemaphoreCollector` without initializing the ddup module first. When the profiler tried to push sample data via handle.push_acquire(), it crashed with a segfault because the C extension's internal state (memory buffers, output file handles) was uninitialized. ## Changes * Added ddup initialization to test_semaphore_and_bounded_semaphore_collectors_coexist (the fix itself) * Extracted init_ddup() helper into new tests/profiling/collector/test_utils.py module (drive-by) * Updated 3 tests to use the shared helper, reducing code duplication (drive-by) ## Testing * CI --- tests/profiling/collector/test_threading.py | 91 +++++++++------------ tests/profiling/collector/test_utils.py | 21 +++++ 2 files changed, 61 insertions(+), 51 deletions(-) create mode 100644 tests/profiling/collector/test_utils.py diff --git a/tests/profiling/collector/test_threading.py b/tests/profiling/collector/test_threading.py index d88c097e492..1360f21b68f 100644 --- a/tests/profiling/collector/test_threading.py +++ b/tests/profiling/collector/test_threading.py @@ -55,6 +55,7 @@ ThreadingLockCollector, ThreadingRLockCollector, ThreadingSemaphoreCollector, ThreadingBoundedSemaphoreCollector ] + # Module-level globals for testing global lock profiling _test_global_lock: LockTypeInst @@ -422,18 +423,10 @@ def test_assertion_error_raised_with_enable_asserts(): import mock import pytest - from ddtrace.internal.datadog.profiling import ddup from ddtrace.profiling.collector.threading import ThreadingLockCollector + from tests.profiling.collector.test_utils import init_ddup - # Initialize ddup (required before using collectors) - assert ddup.is_available, "ddup is not available" - ddup.config( - env="test", - service="test_asserts", - version="1.0", - output_filename="/tmp/test_asserts", - ) - ddup.start() + init_ddup("test_asserts") with ThreadingLockCollector(capture_pct=100): lock = threading.Lock() @@ -456,18 +449,10 @@ def test_all_exceptions_suppressed_by_default() -> None: import mock - from ddtrace.internal.datadog.profiling import ddup from ddtrace.profiling.collector.threading import ThreadingLockCollector + from tests.profiling.collector.test_utils import init_ddup - # Initialize ddup (required before using collectors) - assert ddup.is_available, "ddup is not available" - ddup.config( - env="test", - service="test_exceptions", - version="1.0", - output_filename="/tmp/test_exceptions", - ) - ddup.start() + init_ddup("test_exceptions") with ThreadingLockCollector(capture_pct=100): lock = threading.Lock() @@ -488,6 +473,41 @@ def test_all_exceptions_suppressed_by_default() -> None: lock.release() +def test_semaphore_and_bounded_semaphore_collectors_coexist() -> None: + """Test that Semaphore and BoundedSemaphore collectors can run simultaneously. + + Tests proper patching where inheritance is involved if both parent and child classes are patched, + e.g. when BoundedSemaphore's c-tor calls Semaphore c-tor. + We expect that the call to Semaphore c-tor goes to the unpatched version, and NOT our patched version. + """ + from tests.profiling.collector.test_utils import init_ddup + + init_ddup("test_semaphore_and_bounded_semaphore_collectors_coexist") + + # Both collectors active at the same time - this triggers the inheritance case + with ThreadingSemaphoreCollector(capture_pct=100), ThreadingBoundedSemaphoreCollector(capture_pct=100): + sem = threading.Semaphore(2) + sem.acquire() + sem.release() + + bsem = threading.BoundedSemaphore(3) + bsem.acquire() + bsem.release() + + # If inheritance delegation failed, these attributes will be missing. + wrapped_bsem = bsem.__wrapped__ + assert hasattr(wrapped_bsem, "_cond"), "BoundedSemaphore._cond not initialized (inheritance bug)" + assert hasattr(wrapped_bsem, "_value"), "BoundedSemaphore._value not initialized (inheritance bug)" + assert hasattr(wrapped_bsem, "_initial_value"), "BoundedSemaphore._initial_value not initialized" + + # Verify BoundedSemaphore behavior is preserved (i.e. it raises on over-release) + bsem2 = threading.BoundedSemaphore(1) + bsem2.acquire() + bsem2.release() + with pytest.raises(ValueError, match="Semaphore released too many times"): + bsem2.release() + + class BaseThreadingLockCollectorTest: # These should be implemented by child classes @property @@ -1595,34 +1615,3 @@ def test_bounded_behavior_preserved(self) -> None: # This proves our profiling wrapper doesn't break BoundedSemaphore's behavior with pytest.raises(ValueError, match="Semaphore released too many times"): sem.release() - - -def test_semaphore_and_bounded_semaphore_collectors_coexist() -> None: - """Test that Semaphore and BoundedSemaphore collectors can run simultaneously. - - Tests proper patching where inheritance is involved if both parent and child classes are patched, - e.g. when BoundedSemaphore's c-tor calls Semaphore c-tor. - We expect that the call to Semaphore c-tor goes to the unpatched version, and NOT our patched version. - """ - # Both collectors active at the same time - this triggers the inheritance case - with ThreadingSemaphoreCollector(capture_pct=100), ThreadingBoundedSemaphoreCollector(capture_pct=100): - sem = threading.Semaphore(2) - sem.acquire() - sem.release() - - bsem = threading.BoundedSemaphore(3) - bsem.acquire() - bsem.release() - - # If inheritance delegation failed, these attributes will be missing. - wrapped_bsem = bsem.__wrapped__ - assert hasattr(wrapped_bsem, "_cond"), "BoundedSemaphore._cond not initialized (inheritance bug)" - assert hasattr(wrapped_bsem, "_value"), "BoundedSemaphore._value not initialized (inheritance bug)" - assert hasattr(wrapped_bsem, "_initial_value"), "BoundedSemaphore._initial_value not initialized" - - # Verify BoundedSemaphore behavior is preserved (i.e. it raises on over-release) - bsem2 = threading.BoundedSemaphore(1) - bsem2.acquire() - bsem2.release() - with pytest.raises(ValueError, match="Semaphore released too many times"): - bsem2.release() diff --git a/tests/profiling/collector/test_utils.py b/tests/profiling/collector/test_utils.py new file mode 100644 index 00000000000..43d366d1a1b --- /dev/null +++ b/tests/profiling/collector/test_utils.py @@ -0,0 +1,21 @@ +"""Shared utilities for profiling collector tests.""" + +from ddtrace.internal.datadog.profiling import ddup + + +def init_ddup(test_name: str) -> None: + """Initialize ddup for profiling tests. + + Must be called before using any lock collectors. + + Args: + test_name: Name of the test, used for service name and output filename. + """ + assert ddup.is_available, "ddup is not available" + ddup.config( + env="test", + service=test_name, + version="1.0", + output_filename="/tmp/" + test_name, + ) + ddup.start() From 959fc284f2cf13f3b5580b4cf2fdeccb935cf3be Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Sat, 6 Dec 2025 00:52:13 -0800 Subject: [PATCH 47/86] chore: fix docs builds (#15533) --- docs/spelling_wordlist.txt | 2 ++ scripts/docs/build.sh | 11 +++-------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 49db217e1ac..71b4d4f6414 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -349,3 +349,5 @@ xfail yaaredis openai-agents validators +jitter +durations \ No newline at end of file diff --git a/scripts/docs/build.sh b/scripts/docs/build.sh index bdc101cb515..94874676eda 100755 --- a/scripts/docs/build.sh +++ b/scripts/docs/build.sh @@ -2,18 +2,13 @@ set -eux if [[ "${READTHEDOCS:-}" = "True" ]]; then - # We skip here because we do not check spelling in RTD - echo "Skipping install" + echo "Skipping spelling check in RTD" else if [[ "$(uname)" == "Darwin" ]]; then brew install enchant + export PYENCHANT_LIBRARY_PATH=$(brew --prefix enchant)/lib/libenchant-2.dylib fi + sphinx-build -vvv -W -b spelling docs docs/_build/spelling fi - -if [[ "$(uname)" == "Darwin" ]]; then - export PYENCHANT_LIBRARY_PATH=/opt/homebrew/lib/libenchant-2.dylib -fi - reno lint -sphinx-build -vvv -W -b spelling docs docs/_build/html sphinx-build -vvv -W -b html docs docs/_build/html From 94f7d8ae949fc7594d62c7ca1e9d4b3c1dc6e722 Mon Sep 17 00:00:00 2001 From: Christophe Papazian <114495376+christophe-papazian@users.noreply.github.com> Date: Mon, 8 Dec 2025 09:50:57 +0100 Subject: [PATCH 48/86] chore(aap): improve api10 redirection analysis (#15529) ## Description Add redirection response waf analysis both to urllib and urllib3 ## Testing This will be tested by https://github.com/DataDog/system-tests/pull/5831 APPSEC-60065 --- ddtrace/appsec/_common_module_patches.py | 43 +++++++++++++++++++++--- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/ddtrace/appsec/_common_module_patches.py b/ddtrace/appsec/_common_module_patches.py index 50253370207..4a863070fbc 100644 --- a/ddtrace/appsec/_common_module_patches.py +++ b/ddtrace/appsec/_common_module_patches.py @@ -59,6 +59,7 @@ def _(module): try_wrap_function_wrapper("builtins", "open", wrapped_open_CFDDB7ABBA9081B6) try_wrap_function_wrapper("urllib.request", "OpenerDirector.open", wrapped_open_ED4CF71136E15EBF) try_wrap_function_wrapper("http.client", "HTTPConnection.request", wrapped_request) + try_wrap_function_wrapper("http.client", "HTTPConnection.getresponse", wrapped_response) core.on("asm.block.dbapi.execute", execute_4C9BAC8E228EB347) log.debug("Patching common modules: builtins and urllib.request") _is_patched = True @@ -74,6 +75,8 @@ def unpatch_common_modules(): try_unwrap("urllib3.request", "RequestMethods.request") try_unwrap("builtins", "open") try_unwrap("urllib.request", "OpenerDirector.open") + try_unwrap("http.client", "HTTPConnection.request") + try_unwrap("http.client", "HTTPConnection.getresponse") try_unwrap("_io", "BytesIO.read") try_unwrap("_io", "StringIO.read") subprocess_patch.unpatch() @@ -190,6 +193,26 @@ def wrapped_request(original_request_callable, instance, args, kwargs): return original_request_callable(*args, **kwargs) +def wrapped_response(original_response_callable, instance, args, kwargs): + from ddtrace.appsec._asm_request_context import call_waf_callback + + response = original_response_callable(*args, *kwargs) + env = _get_asm_context() + try: + if _get_rasp_capability("ssrf") and response.__class__.__name__ == "HTTPResponse" and env is not None: + status = response.getcode() + if 300 <= status < 400: + # api10 for redirected response status and headers in urllib + addresses = { + "DOWN_RES_STATUS": str(status), + "DOWN_RES_HEADERS": _build_headers(response.getheaders()), + } + call_waf_callback(addresses, rule_type=EXPLOIT_PREVENTION.TYPE.SSRF_RES) + except Exception: + pass # nosec + return response + + def _parse_http_response_body(response): try: if response.length and response.headers.get("content-type", None) == "application/json": @@ -228,7 +251,7 @@ def wrapped_open_ED4CF71136E15EBF(original_open_callable, instance, args, kwargs try: response = original_open_callable(*args, **kwargs) # api10 response handler for regular responses - if response.__class__.__name__ == "HTTPResponse": + if response.__class__.__name__ == "HTTPResponse" and not (300 <= response.status < 400): addresses = { "DOWN_RES_STATUS": str(response.status), "DOWN_RES_HEADERS": _build_headers(response.getheaders()), @@ -271,7 +294,8 @@ def wrapped_urllib3_make_request(original_request_callable, instance, args, kwar full_url = core.get_item("full_url") env = _get_asm_context() - if _get_rasp_capability("ssrf") and full_url is not None and env is not None: + do_rasp = _get_rasp_capability("ssrf") and full_url is not None and env is not None + if do_rasp: use_body = core.get_item("use_body", False) method = args[1] if len(args) > 1 else kwargs.get("method", None) body = args[3] if len(args) > 3 else kwargs.get("body", None) @@ -292,7 +316,18 @@ def wrapped_urllib3_make_request(original_request_callable, instance, args, kwar core.discard_item("full_url") if res and _must_block(res.actions): raise BlockingException(get_blocked(), EXPLOIT_PREVENTION.BLOCKING, EXPLOIT_PREVENTION.TYPE.SSRF, full_url) - return original_request_callable(*args, **kwargs) + response = original_request_callable(*args, **kwargs) + try: + if do_rasp and response.__class__.__name__ == "BaseHTTPResponse" and 300 <= response.status < 400: + # api10 for redirected response status and headers in urllib3 + addresses = { + "DOWN_RES_STATUS": str(response.status), + "DOWN_RES_HEADERS": response.headers, + } + call_waf_callback(addresses, rule_type=EXPLOIT_PREVENTION.TYPE.SSRF_RES) + except Exception: + pass # nosec + return response def wrapped_urllib3_urlopen(original_open_callable, instance, args, kwargs): @@ -329,7 +364,7 @@ def wrapped_request_D8CB81E472AF98A2(original_request_callable, instance, args, # API10, doing all request calls in HTTPConnection.request try: response = original_request_callable(*args, **kwargs) - if response.__class__.__name__ == "Response": + if response.__class__.__name__ == "Response" and not (300 <= response.status_code < 400): addresses = { "DOWN_RES_STATUS": str(response.status_code), "DOWN_RES_HEADERS": dict(response.headers), From 49663716bd1f73d20b2ff48dc77baaa30d75b7a1 Mon Sep 17 00:00:00 2001 From: Christophe Papazian <114495376+christophe-papazian@users.noreply.github.com> Date: Mon, 8 Dec 2025 11:18:17 +0100 Subject: [PATCH 49/86] chore(tools): allow args of run-tests to be passed to riot (#15525) ## Description previously running the script `./scripts/run-tests` with arguments `-- [any remaining arguments here]` would send all the remaining arguments to pytest via riot. So previously we could do `./scripts/run-tests test_file.py -- -k test_name` With this change, the arguments are passed first to riot allowing for example `./scripts/run-tests test_file.py -- -s -- -k test_name` here the `-s` option is passed to riot (allowing much faster test start step, as it won't recompile everything, (but, most of the time, it will only works if you already run the test without `-s`) and then `-k test_name` is used as a pytest option. If you want the previous behavior, you have to add 2 `-- --` to pass everything to pytest. `./scripts/run-tests test_file.py -- -- -k test_name` ## Motivation `-s` option is a big time saver when investigating a problem. Other riot options may be useful too. --------- Co-authored-by: T. Kowalski --- .claude/skills/run-tests/SKILL.md | 10 +++++----- docs/contributing-testing.rst | 9 +++++++-- scripts/run-tests | 10 ++++++++-- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.claude/skills/run-tests/SKILL.md b/.claude/skills/run-tests/SKILL.md index 3a818fe0f1c..e8d43f4b234 100644 --- a/.claude/skills/run-tests/SKILL.md +++ b/.claude/skills/run-tests/SKILL.md @@ -84,7 +84,7 @@ When you modify files like: #### For Test-Only Changes When you modify `tests/` files (but not test infrastructure): - Run only the specific test files/functions modified -- Use pytest args: `-- -k test_name` or direct test file paths +- Use pytest args: `-- -- -k test_name` or direct test file paths #### For Test Infrastructure Changes When you modify: @@ -121,7 +121,7 @@ This will: For re-running specific tests: ```bash -scripts/run-tests --venv -- -vv -k test_name +scripts/run-tests --venv -- -- -vv -k test_name ``` ## When Tests Fail @@ -243,7 +243,7 @@ scripts/run-tests --list tests/contrib/flask/test_views.py # Output shows: contrib::flask suite # Run just the specific test: -scripts/run-tests --venv flask_py311 -- -vv tests/contrib/flask/test_views.py +scripts/run-tests --venv flask_py311 -- -- -vv tests/contrib/flask/test_views.py ``` ### Example 4: Iterating on a Failing Test @@ -251,7 +251,7 @@ scripts/run-tests --venv flask_py311 -- -vv tests/contrib/flask/test_views.py First run shows one test failing: ```bash -scripts/run-tests --venv flask_py311 -- -vv -k test_view_called_twice +scripts/run-tests --venv flask_py311 -- -- -vv -k test_view_called_twice # Focused on the specific failing test with verbose output ``` @@ -330,7 +330,7 @@ The `scripts/run-tests` system: - Uses `riot` to manage multiple Python/package combinations as venvs - Each venv is a self-contained environment - Docker services are managed per suite lifecycle -- Tests can pass optional pytest arguments with `--` +- Tests can pass optional pytest arguments with `-- --` ### Supported Suite Types diff --git a/docs/contributing-testing.rst b/docs/contributing-testing.rst index 952dac6fc31..dd62a9671ef 100644 --- a/docs/contributing-testing.rst +++ b/docs/contributing-testing.rst @@ -81,9 +81,14 @@ The ``scripts/run-tests`` script handles this automatically: .. code-block:: bash + # Add riot arguments to avoid unnecessary compilation + $ scripts/run-tests tests/contrib/django/ -- -s + # Add pytest arguments for test selection - $ scripts/run-tests tests/contrib/django/ -- -k test_specific_function - $ scripts/run-tests ddtrace/contrib/django/patch.py -- -vvv -s --tb=short + $ scripts/run-tests tests/contrib/django/ -- -- -k test_specific_function + + # Add both riot (first) and pytest (second) arguments + $ scripts/run-tests ddtrace/contrib/django/patch.py -- -s -- -vvv -s --tb=short # Run specific test functions $ scripts/run-tests tests/contrib/flask/ -- -k "test_request or test_response" diff --git a/scripts/run-tests b/scripts/run-tests index c3a551cd9af..a0282726728 100755 --- a/scripts/run-tests +++ b/scripts/run-tests @@ -523,7 +523,7 @@ class TestRunner: # Add riot args if provided if riot_args: - cmd.extend(["--"] + riot_args) + cmd.extend(riot_args) if dry_run: print(f"[DRY RUN] Would execute: {' '.join(cmd)}") @@ -655,8 +655,14 @@ Examples: # Show what would be run without executing scripts/run-tests --dry-run + # Pass additional arguments to riot + scripts/run-tests ddtrace/contrib/django/patch.py -- -s + # Pass additional arguments to pytest - scripts/run-tests ddtrace/contrib/django/patch.py -- -vvv -s --tb=short + scripts/run-tests ddtrace/contrib/django/patch.py -- -- -vvv -s --tb=short + + # Pass additional arguments to riot (first) and pytest (second) + scripts/run-tests ddtrace/contrib/django/patch.py -- -s -- -vvv -k api --tb=short """, ) From 5c567c8103f7456e4c22501bb04a5e220c4150f1 Mon Sep 17 00:00:00 2001 From: Christophe Papazian <114495376+christophe-papazian@users.noreply.github.com> Date: Mon, 8 Dec 2025 15:38:39 +0100 Subject: [PATCH 50/86] chore(ci): update system tests and add small scenario (#15548) ## Description To make sure all api10 features are properly tested, add the APPSEC_RASP_NO_BLOCKING scenario to ddtrace CI. APPSEC-60065 --- .github/workflows/system-tests.yml | 10 +++++++--- .gitlab-ci.yml | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 091dd4bad30..ce0af5f36ee 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -45,7 +45,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: '31bc180ce87184c996400361ddb17c8743eec18f' + ref: '94529f681dcaf74382ed47c3b0c85acdb775b6c9' - name: Download wheels to binaries directory uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 @@ -90,7 +90,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: '31bc180ce87184c996400361ddb17c8743eec18f' + ref: '94529f681dcaf74382ed47c3b0c85acdb775b6c9' - name: Build runner uses: ./.github/actions/install_runner @@ -217,6 +217,10 @@ jobs: if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_RASP + - name: Run APPSEC_RASP_NON_BLOCKING + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1' + run: ./run.sh APPSEC_RASP_NON_BLOCKING + - name: Run APPSEC_RASP_WITHOUT_DOWNSTREAM_BODY_ANALYSIS_USING_MAX if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_RASP_WITHOUT_DOWNSTREAM_BODY_ANALYSIS_USING_MAX @@ -283,7 +287,7 @@ jobs: persist-credentials: false repository: 'DataDog/system-tests' # Automatically managed, use scripts/update-system-tests-version to update - ref: '31bc180ce87184c996400361ddb17c8743eec18f' + ref: '94529f681dcaf74382ed47c3b0c85acdb775b6c9' - name: Download wheels to binaries directory uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ff6b4f0b112..0790f1b3c91 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,7 +15,7 @@ variables: DD_VPA_TEMPLATE: "vpa-template-cpu-p70-10percent-2x-oom-min-cap" # CI_DEBUG_SERVICES: "true" # Automatically managed, use scripts/update-system-tests-version to update - SYSTEM_TESTS_REF: "31bc180ce87184c996400361ddb17c8743eec18f" + SYSTEM_TESTS_REF: "94529f681dcaf74382ed47c3b0c85acdb775b6c9" default: interruptible: true From ae03c06bc7aeccab5753644cd4dce2544263d6f5 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 10:24:15 -0500 Subject: [PATCH 51/86] use new images and refactor a bit --- .gitlab/package.yml | 54 ++++----- .gitlab/scripts/build-wheel-helpers.sh | 154 +++++++++++++++++++++++++ .gitlab/scripts/build-wheel.sh | 139 ++-------------------- 3 files changed, 189 insertions(+), 158 deletions(-) create mode 100755 .gitlab/scripts/build-wheel-helpers.sh diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 99a45cdc439..e382692d721 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -1,6 +1,9 @@ variables: - REGISTRY_BASE_URL: "registry.ddbuild.io/images/mirror/pypa" - IMAGE_TAG: "2025.11.24-1" + REGISTRY_BASE_URL: "registry.ddbuild.io/dd-trace-py" + MANYLINUX2014_AARCH64: "${REGISTRY_BASE_URL}:v85272489-d36e1c8-manylinux2014_aarch64@sha256:7325b77aa61538bd0cfca8902cd1f6758f2297a59375b7cc6c478c7177779648" + MANYLINUX2014_X86_64: "${REGISTRY_BASE_URL}:v85280965-e932d4a-manylinux2014_x86_64@sha256:dfca83eb7235add3ea0450fc15a6016e1164d39c7847a5611fcfd1757c3a474e" + MUSLLINUX_1_2_AARCH64: "${REGISTRY_BASE_URL}:v85281342-e932d4a-musllinux_1_2_aarch64@sha256:771fb5f14156341ff3acd4fd2b8e5b2d02f29e462255ce3671825080c0e4939b" + MUSLLINUX_1_2_X86_64: "${REGISTRY_BASE_URL}:v85281369-e932d4a-musllinux_1_2_x86_64@sha256:258486430b2c422d9cf00a48d9dd01dda308404b8f947155b35b7d3ea697cee9" .PYTHON_VERSIONS: &PYTHON_VERSIONS - "3.9" @@ -18,10 +21,6 @@ variables: - "cp313-cp313" - "cp314-cp314" -.LINUX_OS_IMAGES: &LINUX_OS_IMAGES - - "manylinux2014" - - "musllinux_1_2" - .build_base: stage: package variables: @@ -48,26 +47,38 @@ variables: - "pywheels/*.whl" - "pywheels/*.tar.gz" +"build sdist": + extends: .build_base + image: "${MANYLINUX2014_AARCH64}" + tags: [ "arch:amd64" ] + variables: + DD_CYTHONIZE: "0" + UV_PYTHON: "/opt/python/cp314-cp314/bin/python" + script: + - uv build --sdist --out-dir pywheels/ + "build linux": extends: .build_base - image: ${REGISTRY_BASE_URL}/${LINUX_OS_IMAGE}_${IMAGE_ARCH}:${IMAGE_TAG} + image: ${IMAGE} tags: [ "arch:$ARCH_TAG" ] parallel: matrix: - - IMAGE_ARCH: "x86_64" - ARCH_TAG: "amd64" + - ARCH_TAG: "amd64" PYTHON_TAG: *PYTHON_TAGS - LINUX_OS_IMAGE: *LINUX_OS_IMAGES - - IMAGE_ARCH: "aarch64" - ARCH_TAG: "arm64" + IMAGE: + - "${MANYLINUX2014_AARCH64}" + - "${MUSLLINUX_1_2_AARCH64}" + - ARCH_TAG: "arm64" PYTHON_TAG: *PYTHON_TAGS - LINUX_OS_IMAGE: *LINUX_OS_IMAGES + IMAGE: + - "${MANYLINUX2014_AARCH64}" + - "${MUSLLINUX_1_2_AARCH64}" script: - manylinux-interpreters ensure "${PYTHON_TAG}" - export UV_PYTHON="/opt/python/${PYTHON_TAG}/bin/python" - .gitlab/scripts/build-wheel.sh cache: - - key: v0-build-linux-cache-${ARCH_TAG}-${PYTHON_TAG}-${LINUX_OS_IMAGE} + - key: v0-build-linux-cache-${ARCH_TAG}-${PYTHON_TAG}-${IMAGE} paths: - .cache - .uv @@ -88,7 +99,8 @@ variables: rules: # Skip non-3.14 versions on PRs/feature branches - if: '($IS_MAIN_BRANCH == "false") && ($IS_RELEASE_BRANCH == "false") && ($IS_RELEASE == "false") && ($UV_PYTHON != "3.14")' - when: never + when: manual + - when: always script: - .gitlab/scripts/build-wheel.sh cache: @@ -128,16 +140,6 @@ download_dependency_wheels: needs: - job: "build linux" artifacts: true - parallel: - matrix: - - IMAGE_ARCH: "x86_64" - ARCH_TAG: "amd64" - PYTHON_TAG: *PYTHON_TAGS - LINUX_OS_IMAGE: *LINUX_OS_IMAGES - - IMAGE_ARCH: "aarch64" - ARCH_TAG: "arm64" - PYTHON_TAG: *PYTHON_TAGS - LINUX_OS_IMAGE: *LINUX_OS_IMAGES variables: PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip" parallel: @@ -171,8 +173,6 @@ publish-wheels-to-s3: needs: - job: "build linux" artifacts: true - - job: "build macos" - artifacts: true variables: BUCKET: dd-trace-py-builds script: diff --git a/.gitlab/scripts/build-wheel-helpers.sh b/.gitlab/scripts/build-wheel-helpers.sh new file mode 100755 index 00000000000..9236018368d --- /dev/null +++ b/.gitlab/scripts/build-wheel-helpers.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Helper functions for GitLab CI collapsible sections +section_start() { + echo -e "\e[0Ksection_start:`date +%s`:$1\r\e[0K$2" +} + +section_end() { + echo -e "\e[0Ksection_end:`date +%s`:$1\r\e[0K" +} + + +# Setup Rust (verify/install if needed) +setup_rust() { + section_start "install_rust" "Rust toolchain" + export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}" + if ! command -v rustc &> /dev/null; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + fi + rustup default stable + which rustc && rustc --version + section_end "install_rust" +} + +# Setup sccache (verify/install if needed) +setup_sccache() { + section_start "install_sccache" "sccache" + if ! command -v sccache &> /dev/null; then + # Install openssl-devel for building sccache + if command -v yum &> /dev/null; then + yum install -y openssl-devel + elif command -v apk &> /dev/null; then + apk --no-cache add openssl-dev openssl-libs-static + fi + cargo install sccache + fi + which sccache && sccache --version && sccache --show-stats + section_end "install_sccache" +} + +# Setup Python (verify/install uv if needed) +setup_python() { + section_start "setup_python" "Setting up Python ${UV_PYTHON}" + # Set up PATH for uv and system tools + export PATH="${UV_INSTALL_DIR:-$HOME/.local/bin}:${PATH}" + # If UV_PYTHON is a full path (manylinux), add its bin directory to PATH + if [[ "${UV_PYTHON}" == /* ]]; then + export PATH="$(dirname "${UV_PYTHON}"):${PATH}" + fi + if ! command -v uv &> /dev/null; then + curl -LsSf https://astral.sh/uv/install.sh | sh + fi + which python && python --version + section_end "setup_python" +} + +# Setup directories +setup_env() { + section_start "setup_env" "Setup environment" + PROJECT_DIR="${CI_PROJECT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" + WORK_DIR=$(mktemp -d) + trap "rm -rf '${WORK_DIR}'" EXIT + BUILT_WHEEL_DIR="${WORK_DIR}/built_wheel" + TMP_WHEEL_DIR="${WORK_DIR}/tmp_wheel" + FINAL_WHEEL_DIR="${PROJECT_DIR}/pywheels" + DEBUG_WHEEL_DIR="${PROJECT_DIR}/debugwheelhouse" + mkdir -p "${BUILT_WHEEL_DIR}" "${TMP_WHEEL_DIR}" "${FINAL_WHEEL_DIR}" "${DEBUG_WHEEL_DIR}" + section_end "setup_env" +} + + +build_wheel() { + section_start "build_wheel_function" "Building wheel function" + uv build --wheel --out-dir "${BUILT_WHEEL_DIR}" . + section_end "build_wheel_function" +} + +repair_wheel() { + # Extract debug symbols + section_start "extract_debug_symbols" "Extracting debug symbols" + uv run --no-project scripts/extract_debug_symbols.py "${BUILT_WHEEL_FILE}" --output-dir "${DEBUG_WHEEL_DIR}" + section_end "extract_debug_symbols" + + # Strip wheel + section_start "strip_wheel" "Stripping unneeded files" + uv run --no-project scripts/zip_filter.py "${BUILT_WHEEL_FILE}" \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md + section_end "strip_wheel" + + # List .so files + section_start "list_so_files" "Listing .so files" + unzip -l "${BUILT_WHEEL_FILE}" | grep '\.so$' + section_end "list_so_files" + + # Repair wheel (ONLY PLATFORM-SPECIFIC CODE) + section_start "repair_wheel" "Repairing wheel" + if [[ "$(uname -s)" == "Linux" ]]; then + auditwheel repair -w "${TMP_WHEEL_DIR}" "${BUILT_WHEEL_FILE}" + else + # macOS + MACOSX_DEPLOYMENT_TARGET=14.7 uvx --from="delocate" delocate-wheel \ + --require-archs "${ARCH_TAG}" -w "${TMP_WHEEL_DIR}" -v "${BUILT_WHEEL_FILE}" + fi + section_end "repair_wheel" +} + +setup() {} + setup_env + setup_python + setup_rust + setup_sccache +} + +# Finalize +finalize() { + section_start "finalize_wheel" "Finalizing wheel" + TMP_WHEEL_FILE=$(ls ${TMP_WHEEL_DIR}/*.whl | head -n 1) + mv "${TMP_WHEEL_FILE}" "${FINAL_WHEEL_DIR}/" + FINAL_WHEEL_FILE=$(ls ${FINAL_WHEEL_DIR}/*.whl | head -n 1) + sccache --show-stats + section_end "finalize_wheel" +} + + +# Test wheel +test_wheel() { + section_start "test_wheel" "Testing wheel" + TEST_WHEEL_DIR="${WORK_DIR}/test_wheel" + mkdir -p "${TEST_WHEEL_DIR}" + VENV_PATH="${TEST_WHEEL_DIR}/venv" + uv venv --python="${UV_PYTHON}" "${VENV_PATH}" + export VIRTUAL_ENV="${VENV_PATH}" + export PATH="${VENV_PATH}/bin:${PATH}" + cd "${TEST_WHEEL_DIR}" + uv pip install "${FINAL_WHEEL_FILE}" + + # Diagnostics before running smoke test + echo "=== Environment Diagnostics ===" + echo "VIRTUAL_ENV: ${VIRTUAL_ENV}" + echo "PATH: ${PATH}" + echo "which python: $(which python)" + "${VENV_PATH}/bin/python" --version + echo "=== pip freeze ===" + "${VENV_PATH}/bin/python" -m pip freeze + echo "=== site-packages contents ===" + "${VENV_PATH}/bin/python" -c "import site; print('site-packages:', site.getsitepackages())" + ls -la "${VENV_PATH}/lib/"*/site-packages/ | head -30 + echo "=== Testing direct import ===" + "${VENV_PATH}/bin/python" -c "import ddtrace; print('✓ ddtrace import successful')" || echo "✗ ddtrace import failed" + + # Run smoke test + "${VENV_PATH}/bin/python" "${PROJECT_DIR}/tests/smoke_test.py" + section_end "test_wheel" +} diff --git a/.gitlab/scripts/build-wheel.sh b/.gitlab/scripts/build-wheel.sh index fa0e7115257..0f69eb1a4a7 100755 --- a/.gitlab/scripts/build-wheel.sh +++ b/.gitlab/scripts/build-wheel.sh @@ -1,134 +1,11 @@ #!/usr/bin/env bash set -euo pipefail -# Helper functions for GitLab CI collapsible sections -section_start() { - echo -e "\e[0Ksection_start:`date +%s`:$1\r\e[0K$2" -} - -section_end() { - echo -e "\e[0Ksection_end:`date +%s`:$1\r\e[0K" -} - -# Setup directories -section_start "setup_env" "Setup environment" -PROJECT_DIR="${CI_PROJECT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" -WORK_DIR=$(mktemp -d) -trap "rm -rf '${WORK_DIR}'" EXIT -BUILT_WHEEL_DIR="${WORK_DIR}/built_wheel" -TMP_WHEEL_DIR="${WORK_DIR}/tmp_wheel" -FINAL_WHEEL_DIR="${PROJECT_DIR}/pywheels" -DEBUG_WHEEL_DIR="${PROJECT_DIR}/debugwheelhouse" -mkdir -p "${BUILT_WHEEL_DIR}" "${TMP_WHEEL_DIR}" "${FINAL_WHEEL_DIR}" "${DEBUG_WHEEL_DIR}" -section_end "setup_env" - -# Setup Python (verify/install uv if needed) -section_start "setup_python" "Setting up Python ${UV_PYTHON}" -# Set up PATH for uv and system tools -export PATH="${UV_INSTALL_DIR:-$HOME/.local/bin}:${PATH}" -# If UV_PYTHON is a full path (manylinux), add its bin directory to PATH -if [[ "${UV_PYTHON}" == /* ]]; then - export PATH="$(dirname "${UV_PYTHON}"):${PATH}" -fi -if ! command -v uv &> /dev/null; then - curl -LsSf https://astral.sh/uv/install.sh | sh -fi -which python && python --version -section_end "setup_python" - -# Setup Rust (verify/install if needed) -section_start "install_rust" "Rust toolchain" -export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}" -if ! command -v rustc &> /dev/null; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -fi -rustup default stable -which rustc && rustc --version -section_end "install_rust" - -# Setup sccache (verify/install if needed) -section_start "install_sccache" "sccache" -if ! command -v sccache &> /dev/null; then - # Install openssl-devel for building sccache - if command -v yum &> /dev/null; then - yum install -y openssl-devel - elif command -v apk &> /dev/null; then - apk --no-cache add openssl-dev openssl-libs-static - fi - cargo install sccache -fi -which sccache && sccache --version && sccache --show-stats -section_end "install_sccache" - -# Build wheel -section_start "build_wheel" "Building wheel" -uv build --wheel --out-dir "${BUILT_WHEEL_DIR}" . -BUILT_WHEEL_FILE=$(ls ${BUILT_WHEEL_DIR}/*.whl | head -n 1) -section_end "build_wheel" - -# Extract debug symbols -section_start "extract_debug_symbols" "Extracting debug symbols" -uv run --no-project scripts/extract_debug_symbols.py "${BUILT_WHEEL_FILE}" --output-dir "${DEBUG_WHEEL_DIR}" -section_end "extract_debug_symbols" - -# Strip wheel -section_start "strip_wheel" "Stripping unneeded files" -uv run --no-project scripts/zip_filter.py "${BUILT_WHEEL_FILE}" \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md -section_end "strip_wheel" - -# List .so files -section_start "list_so_files" "Listing .so files" -unzip -l "${BUILT_WHEEL_FILE}" | grep '\.so$' -section_end "list_so_files" - -# Repair wheel (ONLY PLATFORM-SPECIFIC CODE) -section_start "repair_wheel" "Repairing wheel" -if [[ "$(uname -s)" == "Linux" ]]; then - auditwheel repair -w "${TMP_WHEEL_DIR}" "${BUILT_WHEEL_FILE}" -else - # macOS - MACOSX_DEPLOYMENT_TARGET=14.7 uvx --from="delocate" delocate-wheel \ - --require-archs "${ARCH_TAG}" -w "${TMP_WHEEL_DIR}" -v "${BUILT_WHEEL_FILE}" -fi -section_end "repair_wheel" - -# Finalize -section_start "finalize_wheel" "Finalizing wheel" -TMP_WHEEL_FILE=$(ls ${TMP_WHEEL_DIR}/*.whl | head -n 1) -mv "${TMP_WHEEL_FILE}" "${FINAL_WHEEL_DIR}/" -FINAL_WHEEL_FILE=$(ls ${FINAL_WHEEL_DIR}/*.whl | head -n 1) -section_end "finalize_wheel" - -# Test wheel -section_start "test_wheel" "Testing wheel" -TEST_WHEEL_DIR="${WORK_DIR}/test_wheel" -mkdir -p "${TEST_WHEEL_DIR}" -VENV_PATH="${TEST_WHEEL_DIR}/venv" -uv venv --python="${UV_PYTHON}" "${VENV_PATH}" -export VIRTUAL_ENV="${VENV_PATH}" -export PATH="${VENV_PATH}/bin:${PATH}" -cd "${TEST_WHEEL_DIR}" -uv pip install "${FINAL_WHEEL_FILE}" - -# Diagnostics before running smoke test -echo "=== Environment Diagnostics ===" -echo "VIRTUAL_ENV: ${VIRTUAL_ENV}" -echo "PATH: ${PATH}" -echo "which python: $(which python)" -"${VENV_PATH}/bin/python" --version -echo "=== pip freeze ===" -"${VENV_PATH}/bin/python" -m pip freeze -echo "=== site-packages contents ===" -"${VENV_PATH}/bin/python" -c "import site; print('site-packages:', site.getsitepackages())" -ls -la "${VENV_PATH}/lib/"*/site-packages/ | head -30 -echo "=== Testing direct import ===" -"${VENV_PATH}/bin/python" -c "import ddtrace; print('✓ ddtrace import successful')" || echo "✗ ddtrace import failed" - -# Run smoke test -"${VENV_PATH}/bin/python" "${PROJECT_DIR}/tests/smoke_test.py" -section_end "test_wheel" - -# Teardown -section_start "teardown" "Teardown" -sccache --show-stats -section_end "teardown" +# Source helper functions +source "$(dirname "$0")/build-wheel-helpers.sh" + +setup +build_wheel +repair_wheel +finalize +test_wheel From 23c63a805bfbad237a91b9e13750d466397f64fb Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 10:27:37 -0500 Subject: [PATCH 52/86] fix needs --- .gitlab-ci.yml | 10 ++++------ .gitlab/benchmarks/macrobenchmarks.yml | 2 +- .gitlab/benchmarks/microbenchmarks.yml | 6 +++--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0790f1b3c91..8a6d123e2b1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -131,10 +131,9 @@ microbenchmarks: - job: "build linux" parallel: matrix: - - IMAGE_ARCH: "x86_64" - ARCH_TAG: "amd64" + - ARCH_TAG: "amd64" PYTHON_TAG: "cp39-cp39" - LINUX_OS_IMAGE: "manylinux2014" + IMAGE: "registry.ddbuild.io/dd-trace-py:v85280965-e932d4a-manylinux2014_x86_64@sha256:dfca83eb7235add3ea0450fc15a6016e1164d39c7847a5611fcfd1757c3a474e" rules: # Allow failures if explicitly asked - if: $RELEASE_ALLOW_BENCHMARK_FAILURES == "true" @@ -158,10 +157,9 @@ macrobenchmarks: - job: "build linux" parallel: matrix: - - IMAGE_ARCH: "x86_64" - ARCH_TAG: "amd64" + - ARCH_TAG: "amd64" PYTHON_TAG: "cp39-cp39" - LINUX_OS_IMAGE: "manylinux2014" + IMAGE: "registry.ddbuild.io/dd-trace-py:v85280965-e932d4a-manylinux2014_x86_64@sha256:dfca83eb7235add3ea0450fc15a6016e1164d39c7847a5611fcfd1757c3a474e" trigger: include: .gitlab/benchmarks/macrobenchmarks.yml variables: diff --git a/.gitlab/benchmarks/macrobenchmarks.yml b/.gitlab/benchmarks/macrobenchmarks.yml index cd9592c293f..e442e33fe0d 100644 --- a/.gitlab/benchmarks/macrobenchmarks.yml +++ b/.gitlab/benchmarks/macrobenchmarks.yml @@ -27,7 +27,7 @@ candidate: - interruptible: true needs: - pipeline: $PARENT_PIPELINE_ID - job: "build linux: [x86_64, amd64, cp39-cp39, manylinux2014]" + job: "build linux: [amd64, cp39-cp39, registry.ddbuild.io/dd-trace-py:v85280965-e932d4a-manylinux2014_x86_64@sha256:dfca83eb7235add3ea0450fc15a6016e1164d39c7847a5611fcfd1757c3a474e]" artifacts: true script: | cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./ diff --git a/.gitlab/benchmarks/microbenchmarks.yml b/.gitlab/benchmarks/microbenchmarks.yml index 6a0df399689..eab1da777de 100644 --- a/.gitlab/benchmarks/microbenchmarks.yml +++ b/.gitlab/benchmarks/microbenchmarks.yml @@ -145,7 +145,7 @@ candidate: tags: [ "arch:amd64" ] needs: - pipeline: $PARENT_PIPELINE_ID - job: "build linux: [x86_64, amd64, cp39-cp39, manylinux2014]" + job: "build linux: [amd64, cp39-cp39, registry.ddbuild.io/dd-trace-py:v85280965-e932d4a-manylinux2014_x86_64@sha256:dfca83eb7235add3ea0450fc15a6016e1164d39c7847a5611fcfd1757c3a474e]" artifacts: true script: | cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./ @@ -212,8 +212,8 @@ benchmarks-pr-comment: UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for. # Unlike pre-release SLO checks that run unconditionally (which have -# "when: always"), this PR-level check runs only if benchmarks succeed. -# Since benchmark failures block the PR directly, this job blocks the PR if +# "when: always"), this PR-level check runs only if benchmarks succeed. +# Since benchmark failures block the PR directly, this job blocks the PR if # benchmarks succeed but there are SLO breaches. check-slo-breaches: extends: .check-slo-breaches From e1fcd4d4c9380f60622cf119a57b6e386ac1410b Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 10:33:05 -0500 Subject: [PATCH 53/86] change matrix and image tag variables --- .gitlab-ci.yml | 4 ++-- .gitlab/benchmarks/macrobenchmarks.yml | 2 +- .gitlab/benchmarks/microbenchmarks.yml | 2 +- .gitlab/package.yml | 24 ++++++++++++------------ 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8a6d123e2b1..3c6f8adee82 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -133,7 +133,7 @@ microbenchmarks: matrix: - ARCH_TAG: "amd64" PYTHON_TAG: "cp39-cp39" - IMAGE: "registry.ddbuild.io/dd-trace-py:v85280965-e932d4a-manylinux2014_x86_64@sha256:dfca83eb7235add3ea0450fc15a6016e1164d39c7847a5611fcfd1757c3a474e" + IMAGE_TAG: "v85280965-e932d4a-manylinux2014_x86_64" rules: # Allow failures if explicitly asked - if: $RELEASE_ALLOW_BENCHMARK_FAILURES == "true" @@ -159,7 +159,7 @@ macrobenchmarks: matrix: - ARCH_TAG: "amd64" PYTHON_TAG: "cp39-cp39" - IMAGE: "registry.ddbuild.io/dd-trace-py:v85280965-e932d4a-manylinux2014_x86_64@sha256:dfca83eb7235add3ea0450fc15a6016e1164d39c7847a5611fcfd1757c3a474e" + IMAGE_TAG: "v85280965-e932d4a-manylinux2014_x86_64" trigger: include: .gitlab/benchmarks/macrobenchmarks.yml variables: diff --git a/.gitlab/benchmarks/macrobenchmarks.yml b/.gitlab/benchmarks/macrobenchmarks.yml index e442e33fe0d..acfa9a40866 100644 --- a/.gitlab/benchmarks/macrobenchmarks.yml +++ b/.gitlab/benchmarks/macrobenchmarks.yml @@ -27,7 +27,7 @@ candidate: - interruptible: true needs: - pipeline: $PARENT_PIPELINE_ID - job: "build linux: [amd64, cp39-cp39, registry.ddbuild.io/dd-trace-py:v85280965-e932d4a-manylinux2014_x86_64@sha256:dfca83eb7235add3ea0450fc15a6016e1164d39c7847a5611fcfd1757c3a474e]" + job: "build linux: [amd64, cp39-cp39, v85280965-e932d4a-manylinux2014_x86_64]" artifacts: true script: | cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./ diff --git a/.gitlab/benchmarks/microbenchmarks.yml b/.gitlab/benchmarks/microbenchmarks.yml index eab1da777de..64ffb061daa 100644 --- a/.gitlab/benchmarks/microbenchmarks.yml +++ b/.gitlab/benchmarks/microbenchmarks.yml @@ -145,7 +145,7 @@ candidate: tags: [ "arch:amd64" ] needs: - pipeline: $PARENT_PIPELINE_ID - job: "build linux: [amd64, cp39-cp39, registry.ddbuild.io/dd-trace-py:v85280965-e932d4a-manylinux2014_x86_64@sha256:dfca83eb7235add3ea0450fc15a6016e1164d39c7847a5611fcfd1757c3a474e]" + job: "build linux: [amd64, cp39-cp39, v85280965-e932d4a-manylinux2014_x86_64]" artifacts: true script: | cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./ diff --git a/.gitlab/package.yml b/.gitlab/package.yml index e382692d721..e898194cc29 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -1,9 +1,5 @@ variables: REGISTRY_BASE_URL: "registry.ddbuild.io/dd-trace-py" - MANYLINUX2014_AARCH64: "${REGISTRY_BASE_URL}:v85272489-d36e1c8-manylinux2014_aarch64@sha256:7325b77aa61538bd0cfca8902cd1f6758f2297a59375b7cc6c478c7177779648" - MANYLINUX2014_X86_64: "${REGISTRY_BASE_URL}:v85280965-e932d4a-manylinux2014_x86_64@sha256:dfca83eb7235add3ea0450fc15a6016e1164d39c7847a5611fcfd1757c3a474e" - MUSLLINUX_1_2_AARCH64: "${REGISTRY_BASE_URL}:v85281342-e932d4a-musllinux_1_2_aarch64@sha256:771fb5f14156341ff3acd4fd2b8e5b2d02f29e462255ce3671825080c0e4939b" - MUSLLINUX_1_2_X86_64: "${REGISTRY_BASE_URL}:v85281369-e932d4a-musllinux_1_2_x86_64@sha256:258486430b2c422d9cf00a48d9dd01dda308404b8f947155b35b7d3ea697cee9" .PYTHON_VERSIONS: &PYTHON_VERSIONS - "3.9" @@ -21,6 +17,14 @@ variables: - "cp313-cp313" - "cp314-cp314" +.AARCH64_IMAGES: &AARCH64_IMAGES + - "v85272489-d36e1c8-manylinux2014_aarch64" + - "v85281342-e932d4a-musllinux_1_2_aarch64" + +.X86_64_IMAGES: &X86_64_IMAGES + - "v85280965-e932d4a-manylinux2014_x86_64" + - "v85281369-e932d4a-musllinux_1_2_x86_64" + .build_base: stage: package variables: @@ -49,7 +53,7 @@ variables: "build sdist": extends: .build_base - image: "${MANYLINUX2014_AARCH64}" + image: "${IMAGE_BASE_URL}:${MANYLINUX2014_AARCH64}" tags: [ "arch:amd64" ] variables: DD_CYTHONIZE: "0" @@ -59,20 +63,16 @@ variables: "build linux": extends: .build_base - image: ${IMAGE} + image: ${IMAGE_BASE_URL}:${IMAGE_TAG} tags: [ "arch:$ARCH_TAG" ] parallel: matrix: - ARCH_TAG: "amd64" PYTHON_TAG: *PYTHON_TAGS - IMAGE: - - "${MANYLINUX2014_AARCH64}" - - "${MUSLLINUX_1_2_AARCH64}" + IMAGE_TAG: *X86_64_IMAGES - ARCH_TAG: "arm64" PYTHON_TAG: *PYTHON_TAGS - IMAGE: - - "${MANYLINUX2014_AARCH64}" - - "${MUSLLINUX_1_2_AARCH64}" + IMAGE_TAG: *AARCH64_IMAGES script: - manylinux-interpreters ensure "${PYTHON_TAG}" - export UV_PYTHON="/opt/python/${PYTHON_TAG}/bin/python" From a5de6c8136b493caeb00bd136aab21b921bedb97 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 10:37:17 -0500 Subject: [PATCH 54/86] set GH_REPO --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3c6f8adee82..9b89ed7b657 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,6 +51,7 @@ pipeline variables: aud: dd-octo-sts variables: GIT_STRATEGY: none + GH_REPO: DataDog/dd-trace-py script: - | if [ -z ${GH_TOKEN} ] From 49ea22230486c3d2baa2f160aaf399d5f0edbda7 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 10:39:25 -0500 Subject: [PATCH 55/86] fix helper script --- .gitlab/scripts/build-wheel-helpers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/scripts/build-wheel-helpers.sh b/.gitlab/scripts/build-wheel-helpers.sh index 9236018368d..d0d6365ee70 100755 --- a/.gitlab/scripts/build-wheel-helpers.sh +++ b/.gitlab/scripts/build-wheel-helpers.sh @@ -104,7 +104,7 @@ repair_wheel() { section_end "repair_wheel" } -setup() {} +setup() { setup_env setup_python setup_rust From df0a889350d40f46cc55c799da667623ef999706 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 10:43:18 -0500 Subject: [PATCH 56/86] fix ups --- .gitlab/package.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index e898194cc29..e7d2614d5e3 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -1,5 +1,5 @@ variables: - REGISTRY_BASE_URL: "registry.ddbuild.io/dd-trace-py" + IMAGE_BASE_URL: "registry.ddbuild.io/dd-trace-py" .PYTHON_VERSIONS: &PYTHON_VERSIONS - "3.9" @@ -88,17 +88,15 @@ variables: tags: [ "macos:sonoma-$ARCH_TAG" ] parallel: matrix: - - IMAGE_ARCH: "x86_64" - ARCH_TAG: "amd64" + - ARCH_TAG: "amd64" UV_PYTHON: *PYTHON_VERSIONS - - IMAGE_ARCH: "aarch64" - ARCH_TAG: "arm64" + - ARCH_TAG: "arm64" UV_PYTHON: *PYTHON_VERSIONS variables: SYSTEM_VERSION_COMPAT: "0" rules: # Skip non-3.14 versions on PRs/feature branches - - if: '($IS_MAIN_BRANCH == "false") && ($IS_RELEASE_BRANCH == "false") && ($IS_RELEASE == "false") && ($UV_PYTHON != "3.14")' + - if: '($IS_MAIN_BRANCH == "false") && ($IS_RELEASE_BRANCH == "false") && ($IS_RELEASE == "false")' when: manual - when: always script: From d9b07c8cdb09c11c71f518894d7ed98ccd84a7f6 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 11:03:25 -0500 Subject: [PATCH 57/86] fixes --- .gitlab/package.yml | 6 ++---- .gitlab/scripts/build-wheel-helpers.sh | 29 +++++++++++++------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index e7d2614d5e3..e0eb9a4e14a 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -35,7 +35,6 @@ variables: CIBW_BUILD: "1" # Caching DD_SETUP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/download_cache" - CARGO_HOME: "${CI_PROJECT_DIR}/.cache/cargo" PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip" UV_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/uv" UV_INSTALL_DIR: "${CI_PROJECT_DIR}/.uv/" @@ -53,7 +52,7 @@ variables: "build sdist": extends: .build_base - image: "${IMAGE_BASE_URL}:${MANYLINUX2014_AARCH64}" + image: "${IMAGE_BASE_URL}:v85280965-e932d4a-manylinux2014_x86_64" tags: [ "arch:amd64" ] variables: DD_CYTHONIZE: "0" @@ -95,8 +94,7 @@ variables: variables: SYSTEM_VERSION_COMPAT: "0" rules: - # Skip non-3.14 versions on PRs/feature branches - - if: '($IS_MAIN_BRANCH == "false") && ($IS_RELEASE_BRANCH == "false") && ($IS_RELEASE == "false")' + - if: ($IS_MAIN_BRANCH == "false") && ($IS_RELEASE_BRANCH == "false") && ($IS_RELEASE == "false") when: manual - when: always script: diff --git a/.gitlab/scripts/build-wheel-helpers.sh b/.gitlab/scripts/build-wheel-helpers.sh index d0d6365ee70..b0acbe850f3 100755 --- a/.gitlab/scripts/build-wheel-helpers.sh +++ b/.gitlab/scripts/build-wheel-helpers.sh @@ -20,7 +20,7 @@ setup_rust() { fi rustup default stable which rustc && rustc --version - section_end "install_rust" + section_end "install_rust" } # Setup sccache (verify/install if needed) @@ -58,13 +58,13 @@ setup_python() { # Setup directories setup_env() { section_start "setup_env" "Setup environment" - PROJECT_DIR="${CI_PROJECT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" - WORK_DIR=$(mktemp -d) + export PROJECT_DIR="${CI_PROJECT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" + export WORK_DIR=$(mktemp -d) trap "rm -rf '${WORK_DIR}'" EXIT - BUILT_WHEEL_DIR="${WORK_DIR}/built_wheel" - TMP_WHEEL_DIR="${WORK_DIR}/tmp_wheel" - FINAL_WHEEL_DIR="${PROJECT_DIR}/pywheels" - DEBUG_WHEEL_DIR="${PROJECT_DIR}/debugwheelhouse" + export BUILT_WHEEL_DIR="${WORK_DIR}/built_wheel" + export TMP_WHEEL_DIR="${WORK_DIR}/tmp_wheel" + export FINAL_WHEEL_DIR="${PROJECT_DIR}/pywheels" + export DEBUG_WHEEL_DIR="${PROJECT_DIR}/debugwheelhouse" mkdir -p "${BUILT_WHEEL_DIR}" "${TMP_WHEEL_DIR}" "${FINAL_WHEEL_DIR}" "${DEBUG_WHEEL_DIR}" section_end "setup_env" } @@ -73,6 +73,7 @@ setup_env() { build_wheel() { section_start "build_wheel_function" "Building wheel function" uv build --wheel --out-dir "${BUILT_WHEEL_DIR}" . + export BUILT_WHEEL_FILE=$(ls ${BUILT_WHEEL_DIR}/*.whl | head -n 1) section_end "build_wheel_function" } @@ -101,7 +102,7 @@ repair_wheel() { MACOSX_DEPLOYMENT_TARGET=14.7 uvx --from="delocate" delocate-wheel \ --require-archs "${ARCH_TAG}" -w "${TMP_WHEEL_DIR}" -v "${BUILT_WHEEL_FILE}" fi - section_end "repair_wheel" + section_end "repair_wheel" } setup() { @@ -114,10 +115,10 @@ setup() { # Finalize finalize() { section_start "finalize_wheel" "Finalizing wheel" - TMP_WHEEL_FILE=$(ls ${TMP_WHEEL_DIR}/*.whl | head -n 1) + export TMP_WHEEL_FILE=$(ls ${TMP_WHEEL_DIR}/*.whl | head -n 1) mv "${TMP_WHEEL_FILE}" "${FINAL_WHEEL_DIR}/" - FINAL_WHEEL_FILE=$(ls ${FINAL_WHEEL_DIR}/*.whl | head -n 1) - sccache --show-stats + export FINAL_WHEEL_FILE=$(ls ${FINAL_WHEEL_DIR}/*.whl | head -n 1) + sccache --show-stats section_end "finalize_wheel" } @@ -125,9 +126,9 @@ finalize() { # Test wheel test_wheel() { section_start "test_wheel" "Testing wheel" - TEST_WHEEL_DIR="${WORK_DIR}/test_wheel" + export TEST_WHEEL_DIR="${WORK_DIR}/test_wheel" mkdir -p "${TEST_WHEEL_DIR}" - VENV_PATH="${TEST_WHEEL_DIR}/venv" + export VENV_PATH="${TEST_WHEEL_DIR}/venv" uv venv --python="${UV_PYTHON}" "${VENV_PATH}" export VIRTUAL_ENV="${VENV_PATH}" export PATH="${VENV_PATH}/bin:${PATH}" @@ -150,5 +151,5 @@ test_wheel() { # Run smoke test "${VENV_PATH}/bin/python" "${PROJECT_DIR}/tests/smoke_test.py" - section_end "test_wheel" + section_end "test_wheel" } From 8ef8d716158c0ecad94330db7b0eddc834229cab Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 11:07:40 -0500 Subject: [PATCH 58/86] move sdist building to a script --- .gitlab/package.yml | 2 +- .gitlab/scripts/build-sdist.sh | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100755 .gitlab/scripts/build-sdist.sh diff --git a/.gitlab/package.yml b/.gitlab/package.yml index e0eb9a4e14a..2b13793a79e 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -58,7 +58,7 @@ variables: DD_CYTHONIZE: "0" UV_PYTHON: "/opt/python/cp314-cp314/bin/python" script: - - uv build --sdist --out-dir pywheels/ + - .gitlab/scripts/build-sdist.sh "build linux": extends: .build_base diff --git a/.gitlab/scripts/build-sdist.sh b/.gitlab/scripts/build-sdist.sh new file mode 100755 index 00000000000..c621490dd0c --- /dev/null +++ b/.gitlab/scripts/build-sdist.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Source helper functions +source "$(dirname "$0")/build-wheel-helpers.sh" + +setup + +uv build --sdist --out-dir pywheels/ From ab888117d102e79a88b34e13a0cd24dabd7c3bee Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 11:15:34 -0500 Subject: [PATCH 59/86] fix more --- .gitlab/package.yml | 2 +- .gitlab/scripts/build-wheel-helpers.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 2b13793a79e..d3b558fd01a 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -94,7 +94,7 @@ variables: variables: SYSTEM_VERSION_COMPAT: "0" rules: - - if: ($IS_MAIN_BRANCH == "false") && ($IS_RELEASE_BRANCH == "false") && ($IS_RELEASE == "false") + - if: $IS_MAIN_BRANCH == "false" && $IS_RELEASE_BRANCH == "false" && $IS_RELEASE == "false" when: manual - when: always script: diff --git a/.gitlab/scripts/build-wheel-helpers.sh b/.gitlab/scripts/build-wheel-helpers.sh index b0acbe850f3..ccc45892d70 100755 --- a/.gitlab/scripts/build-wheel-helpers.sh +++ b/.gitlab/scripts/build-wheel-helpers.sh @@ -126,6 +126,7 @@ finalize() { # Test wheel test_wheel() { section_start "test_wheel" "Testing wheel" + export UV_LINK_MODE=copy export TEST_WHEEL_DIR="${WORK_DIR}/test_wheel" mkdir -p "${TEST_WHEEL_DIR}" export VENV_PATH="${TEST_WHEEL_DIR}/venv" @@ -142,7 +143,7 @@ test_wheel() { echo "which python: $(which python)" "${VENV_PATH}/bin/python" --version echo "=== pip freeze ===" - "${VENV_PATH}/bin/python" -m pip freeze + uv pip freeze echo "=== site-packages contents ===" "${VENV_PATH}/bin/python" -c "import site; print('site-packages:', site.getsitepackages())" ls -la "${VENV_PATH}/lib/"*/site-packages/ | head -30 From 59d5b757a1180baccc3f4d3b4306dc8799b45c98 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 11:27:05 -0500 Subject: [PATCH 60/86] fix macos building rules? --- .gitlab/package.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index d3b558fd01a..8ba11e9e6c0 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -94,9 +94,11 @@ variables: variables: SYSTEM_VERSION_COMPAT: "0" rules: - - if: $IS_MAIN_BRANCH == "false" && $IS_RELEASE_BRANCH == "false" && $IS_RELEASE == "false" - when: manual - - when: always + - if: $IS_MAIN_BRANCH == "true" || $IS_RELEASE_BRANCH == "true" || $IS_RELEASE == "true" + when: always + - if: UV_PYTHON == "3.14" + when: always + - when: manual script: - .gitlab/scripts/build-wheel.sh cache: From d7bbab68e62d7c594b494176284b5ae21e8a83cf Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 11:31:49 -0500 Subject: [PATCH 61/86] source then install? --- .gitlab/scripts/build-wheel-helpers.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab/scripts/build-wheel-helpers.sh b/.gitlab/scripts/build-wheel-helpers.sh index ccc45892d70..b03bdf73628 100755 --- a/.gitlab/scripts/build-wheel-helpers.sh +++ b/.gitlab/scripts/build-wheel-helpers.sh @@ -134,7 +134,11 @@ test_wheel() { export VIRTUAL_ENV="${VENV_PATH}" export PATH="${VENV_PATH}/bin:${PATH}" cd "${TEST_WHEEL_DIR}" - uv pip install "${FINAL_WHEEL_FILE}" + # Activate venv and install wheel in a subshell + ( + source "${VENV_PATH}/bin/activate" + uv pip install "${FINAL_WHEEL_FILE}" + ) # Diagnostics before running smoke test echo "=== Environment Diagnostics ===" From 407dfec97bce7a66dff083829aa99d54912dd268 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 11:33:23 -0500 Subject: [PATCH 62/86] fix rule? --- .gitlab/package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 8ba11e9e6c0..3a0cf9e4384 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -96,7 +96,7 @@ variables: rules: - if: $IS_MAIN_BRANCH == "true" || $IS_RELEASE_BRANCH == "true" || $IS_RELEASE == "true" when: always - - if: UV_PYTHON == "3.14" + - if: $UV_PYTHON == "3.14" when: always - when: manual script: From 434fa0d586a2ffe0d578c610a98cf7f9bcc90a24 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 11:42:03 -0500 Subject: [PATCH 63/86] move test sdist to gitlab --- .github/workflows/build_deploy.yml | 62 ------------------------------ .gitlab/package.yml | 21 ++++++++++ 2 files changed, 21 insertions(+), 62 deletions(-) diff --git a/.github/workflows/build_deploy.yml b/.github/workflows/build_deploy.yml index 74da807277f..909cc667a68 100644 --- a/.github/workflows/build_deploy.yml +++ b/.github/workflows/build_deploy.yml @@ -72,65 +72,3 @@ jobs: cibw_build: 'cp39* cp310* cp311* cp312* cp313* cp314*' cibw_skip: 'cp39-win_arm64 cp310-win_arm64 cp314t* *_i686' library_version: ${{ needs.compute_version.outputs.library_version }} - - build_sdist: - needs: [ "compute_version" ] - name: Build source distribution - runs-on: ubuntu-latest - env: - SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE: ${{ needs.compute_version.outputs.library_version }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - # Include all history and tags - with: - persist-credentials: false - fetch-depth: 0 - - uses: actions-rust-lang/setup-rust-toolchain@9d7e65c320fdb52dcd45ffaa68deb6c02c8754d9 # v1.12.0 - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - name: Install Python - with: - python-version: '3.12' - - name: Build sdist - run: | - pip install "setuptools_scm[toml]>=4" "cython" "cmake>=3.24.2,<3.28" "setuptools-rust<2" - # Disable cython extensions to avoid compiling .pyx files - DD_CYTHONIZE=0 python setup.py sdist - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: source-dist - path: dist/*.tar.gz - - test_alpine_sdist: - # alpine is the only environment not built/tested by cibuildwheel - name: Test source distribution on Alpine Linux - needs: [build_sdist] - runs-on: ubuntu-latest - container: - image: python:3.9-alpine - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: source-dist - path: dist - - - name: Install build dependencies - # Rust + Cargo are needed for Cryptography - run: apk add git gcc g++ musl-dev libffi-dev openssl-dev bash rust cargo make cmake patchelf - - - name: Check source package - # https://github.com/theacodes/cmarkgfm/issues/84 - run: | - pip install twine readme_renderer[md] "cmarkgfm<2025.10.20" - twine check dist/*.tar.gz - - name: Install source package - env: - CMAKE_BUILD_PARALLEL_LEVEL: 12 - run: pip install dist/*.tar.gz - - - name: Test the source package - run: python $GITHUB_WORKSPACE/tests/smoke_test.py - # Move out of the workspace to avoid importing ddtrace from the source - working-directory: / diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 3a0cf9e4384..8d1c59b1f39 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -107,6 +107,27 @@ variables: - .cache - .uv +"test sdist": + image: registry.ddbuild.io/images/mirror/python:3.9.6-alpine3.14 + tags: [ "arch:aarch64" ] + stage: package + variables: + CARGO_BUILD_JOBS: "12" + CMAKE_BUILD_PARALLEL_LEVEL: "12" + KUBERNETES_CPU_REQUEST: "6" + KUBERNETES_MEMORY_REQUEST: "8Gi" + KUBERNETES_MEMORY_LIMIT: "8Gi" + needs: + - job: "build sdist" +- artifacts: true + script: + - TMP_DIR=$(mktemp -d) && cd $TMP_DIR + - apk add git gcc g++ musl-dev libffi-dev openssl-dev bash rust cargo make cmake patchelf + - pip install twine readme_renderer[md] "cmarkgfm<2025.10.20" + - twine check ${CI_PROJECT_DIR}/pywheels/*.tar.gz + - pip install ${CI_PROJECT_DIR}/*.tar.gz + - python ${CI_PROJECT_DIR}/tests/smoke_test.py + download_ddtrace_artifacts: image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 tags: [ "arch:amd64" ] From eede4dba45ae67f35735b6be87378acb4e2f55d9 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 11:44:13 -0500 Subject: [PATCH 64/86] remove sccache from sdist build --- .gitlab/scripts/build-sdist.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitlab/scripts/build-sdist.sh b/.gitlab/scripts/build-sdist.sh index c621490dd0c..fd2740dae6b 100755 --- a/.gitlab/scripts/build-sdist.sh +++ b/.gitlab/scripts/build-sdist.sh @@ -4,6 +4,15 @@ set -euo pipefail # Source helper functions source "$(dirname "$0")/build-wheel-helpers.sh" +# Skipped sccache setup for sdist build +setup() { + setup_rust + setup_python + setup_env +} + setup +section_start "build_sdist" "Building source distribution" uv build --sdist --out-dir pywheels/ +section_end "build_sdist" From 6f535cb8ef988a80e11eb3741e8d9ca7e7d46a72 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 11:46:42 -0500 Subject: [PATCH 65/86] unset UV_PYTHON --- .gitlab/scripts/build-wheel-helpers.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/scripts/build-wheel-helpers.sh b/.gitlab/scripts/build-wheel-helpers.sh index b03bdf73628..b0933fe995e 100755 --- a/.gitlab/scripts/build-wheel-helpers.sh +++ b/.gitlab/scripts/build-wheel-helpers.sh @@ -135,7 +135,9 @@ test_wheel() { export PATH="${VENV_PATH}/bin:${PATH}" cd "${TEST_WHEEL_DIR}" # Activate venv and install wheel in a subshell + # Unset UV_PYTHON so uv respects the venv instead of the global setting ( + unset UV_PYTHON source "${VENV_PATH}/bin/activate" uv pip install "${FINAL_WHEEL_FILE}" ) From eaacb167b3f82fe3e671107d56aa6ba189bd17e5 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 11:48:17 -0500 Subject: [PATCH 66/86] fix package.yml --- .gitlab/package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 8d1c59b1f39..3b7988de1e3 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -119,7 +119,7 @@ variables: KUBERNETES_MEMORY_LIMIT: "8Gi" needs: - job: "build sdist" -- artifacts: true + artifacts: true script: - TMP_DIR=$(mktemp -d) && cd $TMP_DIR - apk add git gcc g++ musl-dev libffi-dev openssl-dev bash rust cargo make cmake patchelf From 1b27e624c0c075d2b88f9b6ee4e94e0ace3c3cd2 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 11:55:58 -0500 Subject: [PATCH 67/86] fix test sdist arch --- .gitlab/package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 3b7988de1e3..7fc3eeb904e 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -109,7 +109,7 @@ variables: "test sdist": image: registry.ddbuild.io/images/mirror/python:3.9.6-alpine3.14 - tags: [ "arch:aarch64" ] + tags: [ "arch:arm64" ] stage: package variables: CARGO_BUILD_JOBS: "12" From aa5a69455363c7be2750122bb37616e23ed3e5f9 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 11:59:10 -0500 Subject: [PATCH 68/86] add extra log --- .gitlab/scripts/build-wheel-helpers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/scripts/build-wheel-helpers.sh b/.gitlab/scripts/build-wheel-helpers.sh index b0933fe995e..00ce27b3709 100755 --- a/.gitlab/scripts/build-wheel-helpers.sh +++ b/.gitlab/scripts/build-wheel-helpers.sh @@ -156,7 +156,7 @@ test_wheel() { echo "=== Testing direct import ===" "${VENV_PATH}/bin/python" -c "import ddtrace; print('✓ ddtrace import successful')" || echo "✗ ddtrace import failed" - # Run smoke test + echo "=== Running smoke test ===" "${VENV_PATH}/bin/python" "${PROJECT_DIR}/tests/smoke_test.py" section_end "test_wheel" } From 386180ab354d16b58bc604eed2e273120fc5515a Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 12:02:43 -0500 Subject: [PATCH 69/86] wow, we need more memory? --- .gitlab/package.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 7fc3eeb904e..177c68ccefb 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -43,8 +43,8 @@ variables: SCCACHE_CACHE_SIZE: "2G" # job resource requests KUBERNETES_CPU_REQUEST: "6" - KUBERNETES_MEMORY_REQUEST: "8Gi" - KUBERNETES_MEMORY_LIMIT: "8Gi" + KUBERNETES_MEMORY_REQUEST: "10Gi" + KUBERNETES_MEMORY_LIMIT: "10Gi" artifacts: paths: - "pywheels/*.whl" @@ -115,8 +115,8 @@ variables: CARGO_BUILD_JOBS: "12" CMAKE_BUILD_PARALLEL_LEVEL: "12" KUBERNETES_CPU_REQUEST: "6" - KUBERNETES_MEMORY_REQUEST: "8Gi" - KUBERNETES_MEMORY_LIMIT: "8Gi" + KUBERNETES_MEMORY_REQUEST: "10Gi" + KUBERNETES_MEMORY_LIMIT: "10Gi" needs: - job: "build sdist" artifacts: true From 06d2179885e38de54293d894edbf563bab91db88 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 12:12:30 -0500 Subject: [PATCH 70/86] resolve sdist file --- .gitlab/package.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 177c68ccefb..a0a50e258d9 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -121,11 +121,12 @@ variables: - job: "build sdist" artifacts: true script: + - SDIST_FILE=$(ls ${CI_PROJECT_DIR}/pywheels/*.tar.gz | head -n 1) - TMP_DIR=$(mktemp -d) && cd $TMP_DIR - apk add git gcc g++ musl-dev libffi-dev openssl-dev bash rust cargo make cmake patchelf - pip install twine readme_renderer[md] "cmarkgfm<2025.10.20" - - twine check ${CI_PROJECT_DIR}/pywheels/*.tar.gz - - pip install ${CI_PROJECT_DIR}/*.tar.gz + - twine check ${SDIST_FILE} + - pip install ${SDIST_FILE} - python ${CI_PROJECT_DIR}/tests/smoke_test.py download_ddtrace_artifacts: From 5d4da024f0dba428cc2ee622fe14e8869ecfb495 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 14:40:38 -0500 Subject: [PATCH 71/86] use newer/fixed images --- .gitlab-ci.yml | 4 ++-- .gitlab/benchmarks/macrobenchmarks.yml | 2 +- .gitlab/benchmarks/microbenchmarks.yml | 2 +- .gitlab/package.yml | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9b89ed7b657..5873e54c416 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -134,7 +134,7 @@ microbenchmarks: matrix: - ARCH_TAG: "amd64" PYTHON_TAG: "cp39-cp39" - IMAGE_TAG: "v85280965-e932d4a-manylinux2014_x86_64" + IMAGE_TAG: "v85383392-751efc0-manylinux2014_x86_64" rules: # Allow failures if explicitly asked - if: $RELEASE_ALLOW_BENCHMARK_FAILURES == "true" @@ -160,7 +160,7 @@ macrobenchmarks: matrix: - ARCH_TAG: "amd64" PYTHON_TAG: "cp39-cp39" - IMAGE_TAG: "v85280965-e932d4a-manylinux2014_x86_64" + IMAGE_TAG: "v85383392-751efc0-manylinux2014_x86_64" trigger: include: .gitlab/benchmarks/macrobenchmarks.yml variables: diff --git a/.gitlab/benchmarks/macrobenchmarks.yml b/.gitlab/benchmarks/macrobenchmarks.yml index acfa9a40866..58c9a083d06 100644 --- a/.gitlab/benchmarks/macrobenchmarks.yml +++ b/.gitlab/benchmarks/macrobenchmarks.yml @@ -27,7 +27,7 @@ candidate: - interruptible: true needs: - pipeline: $PARENT_PIPELINE_ID - job: "build linux: [amd64, cp39-cp39, v85280965-e932d4a-manylinux2014_x86_64]" + job: "build linux: [amd64, cp39-cp39, v85383392-751efc0-manylinux2014_x86_64]" artifacts: true script: | cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./ diff --git a/.gitlab/benchmarks/microbenchmarks.yml b/.gitlab/benchmarks/microbenchmarks.yml index 64ffb061daa..0c36ed009d8 100644 --- a/.gitlab/benchmarks/microbenchmarks.yml +++ b/.gitlab/benchmarks/microbenchmarks.yml @@ -145,7 +145,7 @@ candidate: tags: [ "arch:amd64" ] needs: - pipeline: $PARENT_PIPELINE_ID - job: "build linux: [amd64, cp39-cp39, v85280965-e932d4a-manylinux2014_x86_64]" + job: "build linux: [amd64, cp39-cp39, v85383392-751efc0-manylinux2014_x86_64]" artifacts: true script: | cp pywheels/*-cp39-cp39-manylinux*_x86_64*.whl ./ diff --git a/.gitlab/package.yml b/.gitlab/package.yml index a0a50e258d9..b0ceb2032c5 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -18,12 +18,12 @@ variables: - "cp314-cp314" .AARCH64_IMAGES: &AARCH64_IMAGES - - "v85272489-d36e1c8-manylinux2014_aarch64" - - "v85281342-e932d4a-musllinux_1_2_aarch64" + - "v85383414-751efc0-manylinux2014_aarch64" + - "v85383359-751efc0-musllinux_1_2_aarch64" .X86_64_IMAGES: &X86_64_IMAGES - - "v85280965-e932d4a-manylinux2014_x86_64" - - "v85281369-e932d4a-musllinux_1_2_x86_64" + - "v85383392-751efc0-manylinux2014_x86_64" + - "v85383325-751efc0-musllinux_1_2_x86_64" .build_base: stage: package From 767091c1abf1d0bb7f21f5bdd400f96d5e9b83b6 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 14:50:54 -0500 Subject: [PATCH 72/86] refresh cache --- .gitlab/package.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index b0ceb2032c5..239a0a5cb94 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -77,10 +77,11 @@ variables: - export UV_PYTHON="/opt/python/${PYTHON_TAG}/bin/python" - .gitlab/scripts/build-wheel.sh cache: - - key: v0-build-linux-cache-${ARCH_TAG}-${PYTHON_TAG}-${IMAGE} + - key: v1-build-linux-cache-${ARCH_TAG}-${PYTHON_TAG}-${IMAGE} paths: - .cache - .uv + - src/native/target* "build macos": extends: .build_base @@ -102,10 +103,11 @@ variables: script: - .gitlab/scripts/build-wheel.sh cache: - - key: v0-build-macos-cache-${ARCH_TAG}-${UV_PYTHON} + - key: v1-build-macos-cache-${ARCH_TAG}-${UV_PYTHON} paths: - .cache - .uv + - src/native/target* "test sdist": image: registry.ddbuild.io/images/mirror/python:3.9.6-alpine3.14 From 58eeebdb4f00d6fa58f28e55f4185593e67c4662 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Mon, 8 Dec 2025 14:54:39 -0500 Subject: [PATCH 73/86] fix rust install on 3.9 alpine --- .gitlab/package.yml | 7 +++++-- .gitlab/scripts/build-wheel-helpers.sh | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 239a0a5cb94..7d582021ded 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -125,8 +125,11 @@ variables: script: - SDIST_FILE=$(ls ${CI_PROJECT_DIR}/pywheels/*.tar.gz | head -n 1) - TMP_DIR=$(mktemp -d) && cd $TMP_DIR - - apk add git gcc g++ musl-dev libffi-dev openssl-dev bash rust cargo make cmake patchelf - - pip install twine readme_renderer[md] "cmarkgfm<2025.10.20" + - | + apk add curl git gcc g++ musl-dev libffi-dev openssl-dev bash cargo make cmake patchelf + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + export PATH="$HOME/.cargo/bin:$PATH" + pip install twine readme_renderer[md] "cmarkgfm<2025.10.20" - twine check ${SDIST_FILE} - pip install ${SDIST_FILE} - python ${CI_PROJECT_DIR}/tests/smoke_test.py diff --git a/.gitlab/scripts/build-wheel-helpers.sh b/.gitlab/scripts/build-wheel-helpers.sh index 00ce27b3709..f49bfe4189b 100755 --- a/.gitlab/scripts/build-wheel-helpers.sh +++ b/.gitlab/scripts/build-wheel-helpers.sh @@ -16,7 +16,7 @@ setup_rust() { section_start "install_rust" "Rust toolchain" export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}" if ! command -v rustc &> /dev/null; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y fi rustup default stable which rustc && rustc --version From 5bade44347ea10104453485744e37ca6970459dc Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 9 Dec 2025 09:20:16 -0500 Subject: [PATCH 74/86] cache less things --- .gitlab/package.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 7d582021ded..29ee0a08c42 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -77,10 +77,8 @@ variables: - export UV_PYTHON="/opt/python/${PYTHON_TAG}/bin/python" - .gitlab/scripts/build-wheel.sh cache: - - key: v1-build-linux-cache-${ARCH_TAG}-${PYTHON_TAG}-${IMAGE} + - key: v2-build-linux-cache-${ARCH_TAG}-${PYTHON_TAG}-${IMAGE} paths: - - .cache - - .uv - src/native/target* "build macos": @@ -103,10 +101,8 @@ variables: script: - .gitlab/scripts/build-wheel.sh cache: - - key: v1-build-macos-cache-${ARCH_TAG}-${UV_PYTHON} + - key: v2-build-macos-cache-${ARCH_TAG}-${UV_PYTHON} paths: - - .cache - - .uv - src/native/target* "test sdist": From 77e0d378391499f4ee6a2aee24487697ebdf3413 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 9 Dec 2025 09:25:39 -0500 Subject: [PATCH 75/86] cache fewer things --- .gitlab/package.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 29ee0a08c42..49c44ebe19a 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -25,6 +25,10 @@ variables: - "v85383392-751efc0-manylinux2014_x86_64" - "v85383325-751efc0-musllinux_1_2_x86_64" +.CACHE_TARGETS: &CACHE_TARGETS + - .cache/download_cache + - src/native/target* + .build_base: stage: package variables: @@ -35,12 +39,6 @@ variables: CIBW_BUILD: "1" # Caching DD_SETUP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/download_cache" - PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip" - UV_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/uv" - UV_INSTALL_DIR: "${CI_PROJECT_DIR}/.uv/" - DD_USE_SCCACHE: "1" - SCCACHE_DIR: "${CI_PROJECT_DIR}/.cache/sccache" - SCCACHE_CACHE_SIZE: "2G" # job resource requests KUBERNETES_CPU_REQUEST: "6" KUBERNETES_MEMORY_REQUEST: "10Gi" @@ -78,8 +76,7 @@ variables: - .gitlab/scripts/build-wheel.sh cache: - key: v2-build-linux-cache-${ARCH_TAG}-${PYTHON_TAG}-${IMAGE} - paths: - - src/native/target* + paths: *CACHE_TARGETS "build macos": extends: .build_base @@ -102,8 +99,7 @@ variables: - .gitlab/scripts/build-wheel.sh cache: - key: v2-build-macos-cache-${ARCH_TAG}-${UV_PYTHON} - paths: - - src/native/target* + paths: *CACHE_TARGETS "test sdist": image: registry.ddbuild.io/images/mirror/python:3.9.6-alpine3.14 From bd328b8a6bfc66f38ecef3047fd7ae4fba268169 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 9 Dec 2025 10:17:46 -0500 Subject: [PATCH 76/86] consistent mtime of files --- .gitlab/package.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 49c44ebe19a..6fa3481212e 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -27,6 +27,9 @@ variables: .CACHE_TARGETS: &CACHE_TARGETS - .cache/download_cache + - .cache/sccache + - .cache/pip + - .cache/uv - src/native/target* .build_base: @@ -39,6 +42,12 @@ variables: CIBW_BUILD: "1" # Caching DD_SETUP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/download_cache" + PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip" + UV_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/uv" + UV_INSTALL_DIR: "${CI_PROJECT_DIR}/.uv/" + DD_USE_SCCACHE: "1" + SCCACHE_DIR: "${CI_PROJECT_DIR}/.cache/sccache" + SCCACHE_CACHE_SIZE: "2G" # job resource requests KUBERNETES_CPU_REQUEST: "6" KUBERNETES_MEMORY_REQUEST: "10Gi" @@ -73,6 +82,12 @@ variables: script: - manylinux-interpreters ensure "${PYTHON_TAG}" - export UV_PYTHON="/opt/python/${PYTHON_TAG}/bin/python" + - | + find src/native -type f -exec touch -t 202401010000 {} \; + find ddtrace/ -type f -name "*.pyx" -exec touch -t 202401010000 {} \; + find ddtrace/ -type f -name "*.c" -exec touch -t 202401010000 {} \; + find ddtrace/ -type f -name "*.cpp" -exec touch -t 202401010000 {} \; + find ddtrace/ -type f -name "*.h" -exec touch -t 202401010000 {} \; - .gitlab/scripts/build-wheel.sh cache: - key: v2-build-linux-cache-${ARCH_TAG}-${PYTHON_TAG}-${IMAGE} From cb81b656ed4c268155dc86b5e05b503f146f4c63 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 9 Dec 2025 14:22:55 -0500 Subject: [PATCH 77/86] can we cache cargo stuff? --- .gitlab/package.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 6fa3481212e..b2c240f7d34 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -30,6 +30,7 @@ variables: - .cache/sccache - .cache/pip - .cache/uv + - .cache/cargo - src/native/target* .build_base: @@ -52,6 +53,15 @@ variables: KUBERNETES_CPU_REQUEST: "6" KUBERNETES_MEMORY_REQUEST: "10Gi" KUBERNETES_MEMORY_LIMIT: "10Gi" + before_script: + # Copy the cargo cache from the project cache to the cargo home + - | + mkdir -p "${CI_PROJECT_DIR}/.cache/cargo" + cp -R "${CI_PROJECT_DIR}/.cache/cargo/{git,registry}" "${CARGO_HOME:-$HOME/.cargo}/" || true + after_script: + # Copy the cargo cache back to the project cache + - | + cp -R "${CARGO_HOME:-$HOME/.cargo}/{git,registry}" "${CI_PROJECT_DIR}/.cache/cargo" artifacts: paths: - "pywheels/*.whl" From e1abd63c5a6c922d06cb59c1bc5a728117e6663d Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 9 Dec 2025 14:36:30 -0500 Subject: [PATCH 78/86] copy individually --- .gitlab/package.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index b2c240f7d34..0cfc7a642a3 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -57,11 +57,13 @@ variables: # Copy the cargo cache from the project cache to the cargo home - | mkdir -p "${CI_PROJECT_DIR}/.cache/cargo" - cp -R "${CI_PROJECT_DIR}/.cache/cargo/{git,registry}" "${CARGO_HOME:-$HOME/.cargo}/" || true + cp -R "${CI_PROJECT_DIR}/.cache/cargo/git" "${CARGO_HOME:-$HOME/.cargo}/" || true + cp -R "${CI_PROJECT_DIR}/.cache/cargo/registry" "${CARGO_HOME:-$HOME/.cargo}/" || true after_script: # Copy the cargo cache back to the project cache - | - cp -R "${CARGO_HOME:-$HOME/.cargo}/{git,registry}" "${CI_PROJECT_DIR}/.cache/cargo" + cp -R "${CARGO_HOME:-$HOME/.cargo}/git" "${CI_PROJECT_DIR}/.cache/cargo" || true + cp -R "${CARGO_HOME:-$HOME/.cargo}/registry" "${CI_PROJECT_DIR}/.cache/cargo" || true artifacts: paths: - "pywheels/*.whl" From c32908e1a09591f63642738652fcc5da7e3c2764 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 9 Dec 2025 14:53:36 -0500 Subject: [PATCH 79/86] split and reduce cache sizes --- .gitlab/package.yml | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 0cfc7a642a3..be78b4e5700 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -25,14 +25,6 @@ variables: - "v85383392-751efc0-manylinux2014_x86_64" - "v85383325-751efc0-musllinux_1_2_x86_64" -.CACHE_TARGETS: &CACHE_TARGETS - - .cache/download_cache - - .cache/sccache - - .cache/pip - - .cache/uv - - .cache/cargo - - src/native/target* - .build_base: stage: package variables: @@ -53,17 +45,6 @@ variables: KUBERNETES_CPU_REQUEST: "6" KUBERNETES_MEMORY_REQUEST: "10Gi" KUBERNETES_MEMORY_LIMIT: "10Gi" - before_script: - # Copy the cargo cache from the project cache to the cargo home - - | - mkdir -p "${CI_PROJECT_DIR}/.cache/cargo" - cp -R "${CI_PROJECT_DIR}/.cache/cargo/git" "${CARGO_HOME:-$HOME/.cargo}/" || true - cp -R "${CI_PROJECT_DIR}/.cache/cargo/registry" "${CARGO_HOME:-$HOME/.cargo}/" || true - after_script: - # Copy the cargo cache back to the project cache - - | - cp -R "${CARGO_HOME:-$HOME/.cargo}/git" "${CI_PROJECT_DIR}/.cache/cargo" || true - cp -R "${CARGO_HOME:-$HOME/.cargo}/registry" "${CI_PROJECT_DIR}/.cache/cargo" || true artifacts: paths: - "pywheels/*.whl" @@ -102,8 +83,15 @@ variables: find ddtrace/ -type f -name "*.h" -exec touch -t 202401010000 {} \; - .gitlab/scripts/build-wheel.sh cache: - - key: v2-build-linux-cache-${ARCH_TAG}-${PYTHON_TAG}-${IMAGE} - paths: *CACHE_TARGETS + - key: v0-build-linux-download-cache-${PYTHON_TAG}-${IMAGE_TAG} + paths: + - .cache/download_cache + - key: v0-build-linux-sccache-${PYTHON_TAG}-${IMAGE_TAG} + paths: + - .cache/sccache + - key: v0-build-linux-native-target-${PYTHON_TAG}-${IMAGE_TAG} + paths: + - src/native/target* "build macos": extends: .build_base @@ -125,8 +113,15 @@ variables: script: - .gitlab/scripts/build-wheel.sh cache: - - key: v2-build-macos-cache-${ARCH_TAG}-${UV_PYTHON} - paths: *CACHE_TARGETS + - key: v0-build-macos-download-cache-${UV_PYTHON}-${ARCH_TAG} + paths: + - .cache/download_cache + - key: v0-build-macos-sccache-${UV_PYTHON}-${ARCH_TAG} + paths: + - .cache/sccache + - key: v0-build-macos-native-target-${UV_PYTHON}-${ARCH_TAG} + paths: + - src/native/target* "test sdist": image: registry.ddbuild.io/images/mirror/python:3.9.6-alpine3.14 From ca3767ed00fb7bdbdfd3917c5b8d33f4861651be Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 9 Dec 2025 15:09:00 -0500 Subject: [PATCH 80/86] try with no caches at all --- .gitlab/package.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index be78b4e5700..cb90dfeab43 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -33,14 +33,6 @@ variables: CMAKE_ARGS: "-DNATIVE_TESTING=OFF" # DEV: Hack to make ASM CMake able to find the Python3 library CIBW_BUILD: "1" - # Caching - DD_SETUP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/download_cache" - PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip" - UV_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/uv" - UV_INSTALL_DIR: "${CI_PROJECT_DIR}/.uv/" - DD_USE_SCCACHE: "1" - SCCACHE_DIR: "${CI_PROJECT_DIR}/.cache/sccache" - SCCACHE_CACHE_SIZE: "2G" # job resource requests KUBERNETES_CPU_REQUEST: "6" KUBERNETES_MEMORY_REQUEST: "10Gi" @@ -82,16 +74,6 @@ variables: find ddtrace/ -type f -name "*.cpp" -exec touch -t 202401010000 {} \; find ddtrace/ -type f -name "*.h" -exec touch -t 202401010000 {} \; - .gitlab/scripts/build-wheel.sh - cache: - - key: v0-build-linux-download-cache-${PYTHON_TAG}-${IMAGE_TAG} - paths: - - .cache/download_cache - - key: v0-build-linux-sccache-${PYTHON_TAG}-${IMAGE_TAG} - paths: - - .cache/sccache - - key: v0-build-linux-native-target-${PYTHON_TAG}-${IMAGE_TAG} - paths: - - src/native/target* "build macos": extends: .build_base @@ -112,16 +94,6 @@ variables: - when: manual script: - .gitlab/scripts/build-wheel.sh - cache: - - key: v0-build-macos-download-cache-${UV_PYTHON}-${ARCH_TAG} - paths: - - .cache/download_cache - - key: v0-build-macos-sccache-${UV_PYTHON}-${ARCH_TAG} - paths: - - .cache/sccache - - key: v0-build-macos-native-target-${UV_PYTHON}-${ARCH_TAG} - paths: - - src/native/target* "test sdist": image: registry.ddbuild.io/images/mirror/python:3.9.6-alpine3.14 From d4f5c162f5c02301b08b33e19bc1c8b0a9fcdcf4 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 9 Dec 2025 15:21:15 -0500 Subject: [PATCH 81/86] skip sccache from macos --- .gitlab/scripts/build-sdist.sh | 6 ------ .gitlab/scripts/build-wheel-helpers.sh | 17 ----------------- 2 files changed, 23 deletions(-) diff --git a/.gitlab/scripts/build-sdist.sh b/.gitlab/scripts/build-sdist.sh index fd2740dae6b..a58700c1b8c 100755 --- a/.gitlab/scripts/build-sdist.sh +++ b/.gitlab/scripts/build-sdist.sh @@ -5,12 +5,6 @@ set -euo pipefail source "$(dirname "$0")/build-wheel-helpers.sh" # Skipped sccache setup for sdist build -setup() { - setup_rust - setup_python - setup_env -} - setup section_start "build_sdist" "Building source distribution" diff --git a/.gitlab/scripts/build-wheel-helpers.sh b/.gitlab/scripts/build-wheel-helpers.sh index f49bfe4189b..11f73b98b76 100755 --- a/.gitlab/scripts/build-wheel-helpers.sh +++ b/.gitlab/scripts/build-wheel-helpers.sh @@ -23,22 +23,6 @@ setup_rust() { section_end "install_rust" } -# Setup sccache (verify/install if needed) -setup_sccache() { - section_start "install_sccache" "sccache" - if ! command -v sccache &> /dev/null; then - # Install openssl-devel for building sccache - if command -v yum &> /dev/null; then - yum install -y openssl-devel - elif command -v apk &> /dev/null; then - apk --no-cache add openssl-dev openssl-libs-static - fi - cargo install sccache - fi - which sccache && sccache --version && sccache --show-stats - section_end "install_sccache" -} - # Setup Python (verify/install uv if needed) setup_python() { section_start "setup_python" "Setting up Python ${UV_PYTHON}" @@ -109,7 +93,6 @@ setup() { setup_env setup_python setup_rust - setup_sccache } # Finalize From 93eebe70bdbc71afb4d2305eca76fba83060e965 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 9 Dec 2025 15:33:43 -0500 Subject: [PATCH 82/86] only download GHA artifacts if we know there will be a build --- .gitlab/package.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index cb90dfeab43..db97ea72f24 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -127,6 +127,12 @@ download_ddtrace_artifacts: id_tokens: DDOCTOSTS_ID_TOKEN: aud: dd-octo-sts + rules: + - if: $IS_MAIN_BRANCH == "true" || $IS_RELEASE_BRANCH == "true" || $IS_RELEASE == "true" + when: always + - if: $HAS_OPEN_PR == "true" + when: always + - when: manual script: | if [ -z ${GH_TOKEN} ] then From d06bea8999afc3ccbe4d0cfa9eee58ab07b38d24 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Tue, 9 Dec 2025 15:46:24 -0500 Subject: [PATCH 83/86] fix gh pr lookup --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5873e54c416..8247321c612 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -67,7 +67,7 @@ pipeline variables: git config --global --add safe.directory "${CI_PROJECT_DIR}" - | # Determine if we have an open GitHub PR for this commit - GH_PR_NUMBER=$(gh pr list --search "${CI_COMMIT_REF_NAME}" --json number --jq '.[0].number' || echo "") + GH_PR_NUMBER=$(gh pr list --search "${CI_COMMIT_SHA}" --json number --jq '.[0].number' || echo "") echo "GH_PR_NUMBER=${GH_PR_NUMBER}" | tee -a workflow.env echo "HAS_OPEN_PR=$(if [ -z "${GH_PR_NUMBER}" ]; then echo "false"; else echo "true"; fi)" | tee -a workflow.env echo "IS_MAIN_BRANCH=$(if [ "${CI_COMMIT_BRANCH}" == "main" ]; then echo "true"; else echo "false"; fi)" | tee -a workflow.env From f782b525fa9e586579a58d987237ab5fc91a86a2 Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Wed, 10 Dec 2025 08:24:29 -0500 Subject: [PATCH 84/86] remove sccache show stats --- .gitlab/scripts/build-wheel-helpers.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab/scripts/build-wheel-helpers.sh b/.gitlab/scripts/build-wheel-helpers.sh index 11f73b98b76..0d88437b1eb 100755 --- a/.gitlab/scripts/build-wheel-helpers.sh +++ b/.gitlab/scripts/build-wheel-helpers.sh @@ -101,7 +101,6 @@ finalize() { export TMP_WHEEL_FILE=$(ls ${TMP_WHEEL_DIR}/*.whl | head -n 1) mv "${TMP_WHEEL_FILE}" "${FINAL_WHEEL_DIR}/" export FINAL_WHEEL_FILE=$(ls ${FINAL_WHEEL_DIR}/*.whl | head -n 1) - sccache --show-stats section_end "finalize_wheel" } From 82db874141703e8f8d84a592a7f360022e93570c Mon Sep 17 00:00:00 2001 From: brettlangdon Date: Wed, 10 Dec 2025 08:40:47 -0500 Subject: [PATCH 85/86] fix gha matrix --- .github/workflows/build_python_3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_python_3.yml b/.github/workflows/build_python_3.yml index 51c05eedeee..2f366c6db1c 100644 --- a/.github/workflows/build_python_3.yml +++ b/.github/workflows/build_python_3.yml @@ -65,7 +65,7 @@ jobs: MATRIX_INCLUDE=$( { cibuildwheel --print-build-identifiers --platform windows --archs AMD64,x86 | jq -cR '{only: ., os: "windows-latest"}' \ - && cibuildwheel --print-build-identifiers --platform windows --archs ARM64 | jq -cR '{only: ., os: "windows-11-arm"}' \ + && cibuildwheel --print-build-identifiers --platform windows --archs ARM64 | jq -cR '{only: ., os: "windows-11-arm"}' } | jq -sc ) echo $MATRIX_INCLUDE From b0b697f0497a0c8c8ba0dd253c0e30184e62d6ae Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Thu, 11 Dec 2025 12:31:14 -0500 Subject: [PATCH 86/86] Apply suggestion from @brettlangdon --- .gitlab/package.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 7f6d0422e7b..a937d3c56e7 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -128,7 +128,11 @@ download_ddtrace_artifacts: DDOCTOSTS_ID_TOKEN: aud: dd-octo-sts rules: - - if: $IS_MAIN_BRANCH == "true" || $IS_RELEASE_BRANCH == "true" || $IS_RELEASE == "true" + - if: $IS_MAIN_BRANCH == "true" + when: always + - if: $IS_RELEASE_BRANCH == "true" + when: always + - if: $IS_RELEASE == "true" when: always - if: $HAS_OPEN_PR == "true" when: always