ci: add precomputed pipeline variables (#15582)

## Description

This PRs adds an initial "setup" stage to the pipeline where we compute
and expose the following variables to the rest of the pipeline:


- **GH_PR_NUMBER:** The GitHub PR number if there is an open PR for this
commit, empty otherwise
- **HAS_OPEN_PR:** "true" if there is an open PR for this commit,
"false" otherwise
- **IS_MAIN_BRANCH:** "true" if the current branch is main
- **IS_RELEASE_BRANCH:** "true" if the current branch is a release
branch (e.g., "1.2"), "false" otherwise
- **IS_RELEASE:** "true" if the current commit is a release tag (e.g.,
"v1.2.3"), "false" otherwise
- **IS_MERGE_QUEUE:** "true" if the current branch is a merge queue
branch (e.g., starts with "gh-readonly-queue/"), "false" otherwise

The goal is to provide consistent controls for which jobs to run when,
e.g. should a job run only when we have a PR open or only during a
release?

real example, with #15572 we want to be sure we still restrict
benchmarks to only run/pass when we have an open PR to reduce volume,
today that is controlled by the fact that the GHA wheel building only
happens when a PR is open.

## Testing

<!-- Describe your testing strategy or note what tests are included -->

## Risks

<!-- Note any risks associated with this change, or "None" if no risks
-->

## Additional Notes

<!-- Any other information that would be helpful for reviewers -->

Author: brettlangdon

.gitlab-ci.yml

@@ -1,4 +1,5 @@
 stages:
+  - setup
   - package
   - tests
   - shared-pipeline
@@ -41,6 +42,50 @@ include:
   - local: ".gitlab/benchmarks/serverless.yml"
   - local: ".gitlab/native.yml"
 
+
+# Expose the following variables to the rest of the pipeline
+#   GH_PR_NUMBER: The GitHub PR number if there is an open PR for this commit, empty otherwise
+#   HAS_OPEN_PR: "true" if there is an open PR for this commit, "false" otherwise
+#   IS_MAIN_BRANCH: "true" if the current branch is main
+#   IS_RELEASE_BRANCH: "true" if the current branch is a release branch (e.g., "1.2"), "false" otherwise
+#   IS_RELEASE: "true" if the current commit is a release tag (e.g., "v1.2.3"), "false" otherwise
+#   IS_MERGE_QUEUE: "true" if the current branch is a merge queue branch (e.g., starts with "gh-readonly-queue/"), "false" otherwise
+pipeline variables:
+  image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
+  tags: [ "arch:amd64" ]
+  stage: setup
+  id_tokens:
+    DDOCTOSTS_ID_TOKEN:
+      aud: dd-octo-sts
+  variables:
+    GIT_STRATEGY: none
+    GH_REPO: DataDog/dd-trace-py
+  script:
+    - |
+      if [ -z ${GH_TOKEN} ]
+      then
+        # Use dd-octo-sts to get GitHub token
+        dd-octo-sts token --scope DataDog/dd-trace-py --policy gitlab.github-access.read > token
+        gh auth login --with-token < token
+        rm token
+      fi
+    - |
+      # Prevent git operation errors:
+      #   failed to determine base repo: failed to run git: fatal: detected dubious ownership in repository at ...
+      git config --global --add safe.directory "${CI_PROJECT_DIR}"
+    - |
+      # Determine if we have an open GitHub PR for this commit
+      GH_PR_NUMBER=$(gh pr list --state open --search "${CI_COMMIT_SHA}" --json number --jq '.[0].number' || echo "")
+      echo "GH_PR_NUMBER=${GH_PR_NUMBER}" | tee -a workflow.env
+      echo "HAS_OPEN_PR=$(if [ -z "${GH_PR_NUMBER}" ]; then echo "false"; else echo "true"; fi)" | tee -a workflow.env
+      echo "IS_MAIN_BRANCH=$(if [ "${CI_COMMIT_BRANCH}" == "main" ]; then echo "true"; else echo "false"; fi)" | tee -a workflow.env
+      echo "IS_RELEASE_BRANCH=$(if [[ "${CI_COMMIT_BRANCH}" =~ ^[0-9]+\.[0-9]+$ ]]; then echo "true"; else echo "false"; fi)" | tee -a workflow.env
+      echo "IS_RELEASE=$(if [[ "${CI_COMMIT_TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(rc[0-9]+)?$ ]]; then echo "true"; else echo "false"; fi)" | tee -a workflow.env
+      echo "IS_MERGE_QUEUE=$(if [[ "${CI_COMMIT_BRANCH}" =~ ^gh-readonly-queue/.*$ ]]; then echo "true"; else echo "false"; fi)" | tee -a workflow.env
+  artifacts:
+    reports:
+      dotenv: workflow.env
+
 tests-gen:
   stage: tests
   extends: .testrunner
@@ -55,14 +100,14 @@ tests-gen:
         export GH_TOKEN=$(dd-octo-sts token --scope DataDog/dd-trace-py --policy gitlab.github-access.read)
       fi
     - scripts/gen_gitlab_config.py --verbose
-  needs: []
+  needs: [ "pipeline variables" ]
   artifacts:
     paths:
       - .gitlab/tests-gen.yml
 
 run-tests-trigger:
  stage: tests
- needs: [ tests-gen ]
+ needs: [ tests-gen, "pipeline variables" ]
  # Allow the child job to fail if explicitly asked
  rules:
    - if: $RELEASE_ALLOW_TEST_FAILURES == "true"
@@ -118,7 +163,7 @@ macrobenchmarks:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
       when: always
-    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+/
+    - if: $IS_RELEASE == "true"
       when: always
     - when: manual
 
@@ -131,10 +176,16 @@ check_new_flaky_tests:
     - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.${CI_PROJECT_NAME}.dd-api-key-qualitygate --with-decryption --query "Parameter.Value" --out text)
     - export DD_APP_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.${CI_PROJECT_NAME}.dd-app-key-qualitygate --with-decryption --query "Parameter.Value" --out text)
     - datadog-ci gate evaluate
-  except:
-    - main
-    - '[0-9].[0-9]*'
-    - 'mq-working-branch**'
+  rules:
+    - if: $IS_MAIN_BRANCH == "true"
+      when: never
+    - if: $IS_RELEASE == "true"
+      when: never
+    - if: $IS_RELEASE_BRANCH == "true"
+      when: never
+    - if: $IS_MERGE_QUEUE == "true"
+      when: never
+    - when: on_success
 
 requirements_json_test:
   rules:
@@ -148,10 +199,10 @@ package-oci:
 
 promote-oci-to-prod:
   stage: release
-  rules: null
-  only:
-    # TODO: Support publishing rc releases
-    - /^v[0-9]+\.[0-9]+\.[0-9]+$/
+  rules:
+    - if: $IS_RELEASE == "true"
+      when: on_success
+    - when: never
   needs:
     - job: release_pypi_prod
     - job: package-oci
@@ -177,10 +228,10 @@ promote-oci-to-staging:
 
 publish-lib-init-pinned-tags:
   stage: release
-  rules: null
-  only:
-    # TODO: Support publishing rc releases
-    - /^v[0-9]+\.[0-9]+\.[0-9]+$/
+  rules:
+    - if: $IS_RELEASE == "true"
+      when: on_success
+    - when: never
   needs:
     - job: release_pypi_prod
     - job: create-multiarch-lib-injection-image

.gitlab/package.yml

@@ -116,11 +116,9 @@ verify_package_version:
   tags: [ "arch:amd64" ]
   stage: package
   needs: [ download_ddtrace_artifacts ]
-  only:
-    # v2.10.0
-    # v2.10.1
-    # v2.10.0rc0
-    # v2.10.0rc5
-    - /^v[0-9]+\.[0-9]+\.[0-9]+(rc[0-9]+)?$/
+  rules:
+    - if: $IS_RELEASE == "true"
+      when: on_success
+    - when: never
   script:
     - .gitlab/verify-package-versions.sh

.gitlab/release.yml

@@ -3,12 +3,10 @@ variables:
 
 .release_base:
   stage: release
-  only:
-    # v2.10.0
-    # v2.10.1
-    # v2.10.0rc0
-    # v2.10.0rc5
-    - /^v[0-9]+\.[0-9]+\.[0-9]+(rc[0-9]+)?$/
+  rules:
+    - if: $IS_RELEASE == "true"
+      when: on_success
+    - when: never
 
 .release_pypi:
   extends: .release_base

.gitlab/templates/build-base-venvs.yml

@@ -12,10 +12,16 @@ build_base_venvs:
     DD_USE_SCCACHE: '1'
     DD_FAST_BUILD: '1'
   rules:
-    - if: '$CI_COMMIT_REF_NAME == "main"'
+    - if: $IS_MAIN == "true"
       variables:
         DD_FAST_BUILD: '0'
-    - when: always
+    - if: $IS_RELEASE == "true"
+      variables:
+        DD_FAST_BUILD: '0'
+    - if: $IS_RELEASE_BRANCH == "true"
+      variables:
+        DD_FAST_BUILD: '0'
+    - when: on_success
   script: |
     set -e -o pipefail
     riot -P -v generate --python=$PYTHON_VERSION

scripts/needs_testrun.py

@@ -159,6 +159,18 @@ def needs_testrun(suite: str, pr_number: int, sha: t.Optional[str] = None) -> bo
     """
     if "itr:noskip" in get_latest_commit_message().lower():
         return True
+
+    # Custom GitLab env variables
+    # Always run all tests under these conditions
+    if os.getenv("IS_MERGE_QUEUE", "false") == "true":
+        return True
+    if os.getenv("IS_MAIN_BRANCH", "false") == "true":
+        return True
+    if os.getenv("IS_RELEASE_BRANCH", "false") == "true":
+        return True
+    if os.getenv("IS_RELEASE", "false") == "true":
+        return True
+
     try:
         patterns = get_patterns(suite)
     except Exception as exc:
@@ -209,6 +221,11 @@ def _get_pr_number() -> int:
     if pr_url is not None:
         return int(pr_url.split("/")[-1])
 
+    # Custom environment variable
+    number = os.environ.get("GH_PR_NUMBER")
+    if number is not None:
+        return int(number)
+
     # GitLab
     ref_name = os.environ.get("CI_COMMIT_REF_NAME")
     if ref_name is not None: