Merge pull request #1774 from roboflow/feature/adjustments-in-e2e-tests-of-inference-exp

E2E platform tests for `inference-exp` to run in CI

Author: PawelPeczek-Roboflow

.github/workflows/e2e_tests_inference_experimental_cpu.yml

@@ -0,0 +1,49 @@
+name: E2E tests - inference_experimental (CPU)
+permissions:
+  contents: read
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  e2e-tests-inference-experimental-gpu:
+    runs-on:
+      labels: depot-ubuntu-22.04-4
+      group: public-depot
+    timeout-minutes: 30
+    steps:
+      - name: 🛎️ Checkout
+        uses: actions/checkout@v4
+      - name: 📦 Cache Python packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-uv-3.12-${{ hashFiles('inference_experimental/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-3.12-
+      - name: 📦 Cache apt packages
+        uses: actions/cache@v3
+        with:
+          path: ~/apt-cache
+          key: ${{ runner.os }}-apt-cuda-12-4
+          restore-keys: |
+            ${{ runner.os }}-apt-cuda-
+      - name: 🐍 Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+          check-latest: true
+      - name: 📦 Install dependencies
+        working-directory: inference_experimental
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install uv
+          python -m uv venv
+          python -m uv pip install -e .[test,onnx-cpu,torch-cpu] --python .venv
+      - name: 🧪 Run E2E tests
+        working-directory: inference_experimental
+        timeout-minutes: 20
+        run: |
+          source .venv/bin/activate
+          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m pytest -m "e2e_model_inference and not gpu_only" tests/e2e_platform_tests

.github/workflows/e2e_tests_inference_experimental_gpu.yml

@@ -0,0 +1,69 @@
+name: E2E tests - inference_experimental (GPU)
+permissions:
+  contents: read
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  e2e-tests-inference-experimental-gpu:
+    runs-on: Roboflow-GPU-VM-Runner
+    timeout-minutes: 30
+    steps:
+      - name: 🛎️ Checkout
+        uses: actions/checkout@v4
+      - name: 📦 Cache Python packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-uv-3.12-${{ hashFiles('inference_experimental/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-3.12-
+      - name: 📦 Cache apt packages
+        uses: actions/cache@v3
+        with:
+          path: ~/apt-cache
+          key: ${{ runner.os }}-apt-cuda-12-4
+          restore-keys: |
+            ${{ runner.os }}-apt-cuda-
+      - name: 🐍 Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+          check-latest: true
+      - name: 🔧 Install CUDA development toolkit
+        run: |
+          mkdir -p ~/apt-cache
+
+          sudo mkdir -p /var/cache/apt/archives
+          sudo cp -r ~/apt-cache/* /var/cache/apt/archives/ 2>/dev/null || true
+
+          sudo apt-get update
+          sudo apt-get install -y \
+              cuda-cudart-dev-12-5 \
+              cuda-nvcc-12-5 \
+              cuda-profiler-api-12-5 \
+              libcurand-dev-12-5 \
+              libvips-dev
+
+          sudo cp /var/cache/apt/archives/*.deb ~/apt-cache/ 2>/dev/null || true
+          sudo chown -R $(whoami):$(whoami) ~/apt-cache
+      - name: 📦 Install dependencies
+        working-directory: inference_experimental
+        run: |
+          export CUDA_HOME=/usr/local/cuda-12.5
+          export PATH=/usr/local/cuda-12.5/bin:$PATH
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.5/targets/x86_64-linux/lib64:$LD_LIBRARY_PATH
+          export CPATH=/usr/local/cuda-12.5/targets/x86_64-linux/include:$CPATH
+          export LIBRARY_PATH=/usr/local/cuda-12.5/targets/x86_64-linux/lib64:$LIBRARY_PATH
+          python -m pip install --upgrade pip
+          python -m pip install uv
+          python -m uv venv
+          python -m uv pip install -e .[test,onnx-cu12,trt10,torch-cu124] --python .venv
+      - name: 🧪 Run E2E tests
+        working-directory: inference_experimental
+        timeout-minutes: 20
+        run: |
+          source .venv/bin/activate
+          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m pytest -m "e2e_model_inference and not cpu_only" tests/e2e_platform_tests

.github/workflows/integration_tests_inference_experimental_cpu.yml

@@ -34,7 +34,7 @@ jobs:
     runs-on:
       labels: depot-ubuntu-22.04-4
       group: public-depot
-    timeout-minutes: 10
+    timeout-minutes: 15
     strategy:
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
@@ -70,7 +70,7 @@ jobs:
       - name: 🧪 Run tests marked `${{ matrix.extras.marker }}`
         if: ${{ (github.event.inputs.extras == '' || github.event.inputs.extras == matrix.extras.marker) && (github.event.inputs.python_version == '' || github.event.inputs.python_version == matrix.python-version) }}
         working-directory: inference_experimental
-        timeout-minutes: 30
+        timeout-minutes: 15
         run: |
           source .venv/bin/activate
           python -m pytest -n ${{ matrix.extras.workers }} -m "${{ matrix.extras.marker }} and not gpu_only" tests/integration_tests

.github/workflows/integration_tests_inference_experimental_gpu.yml

@@ -29,7 +29,7 @@ jobs:
   integration-tests-inference-experimental-gpu:
     name: ${{ matrix.extras.marker }}:${{ matrix.python-version }}
     runs-on: Roboflow-GPU-VM-Runner
-    timeout-minutes: 20
+    timeout-minutes: 30
     strategy:
       matrix:
         python-version: ["3.12"]
@@ -98,7 +98,7 @@ jobs:
       - name: 🧪 Run tests marked `${{ matrix.extras.marker }}`
         if: ${{ (github.event.inputs.extras == '' || github.event.inputs.extras == matrix.extras.marker) && (github.event.inputs.python_version == '' || github.event.inputs.python_version == matrix.python-version) }}
         working-directory: inference_experimental
-        timeout-minutes: 20
+        timeout-minutes: 25
         run: |
           source .venv/bin/activate
           python -m pytest -n ${{ matrix.extras.workers }} -m "${{ matrix.extras.marker }} and not cpu_only" tests/integration_tests

.gitignore

@@ -206,6 +206,6 @@ tests/inference_sdk/unit_tests/http/inference_profiling
 !app_bundles/**/*.spec
 !app_bundles/**/*.png
 inference_experimental/tests/integration_tests/models/assets/
-inference_experimental/tests/integration_tests/e2e/assets/
+inference_experimental/tests/e2e_platform_tests/assets/
 
 inference_testing

inference_experimental/tests/integration_tests/e2e/__init__.py renamed to inference_experimental/tests/e2e_platform_tests/__init__.py

@@ -0,0 +1,42 @@
+import os
+
+import cv2
+import numpy as np
+import pytest
+import requests
+from filelock import FileLock
+
+ASSETS_DIR = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "assets")
+)
+DOG_IMAGE_PATH = os.path.join(ASSETS_DIR, "images", "dog.jpeg")
+DOG_IMAGE_URL = (
+    "https://storage.googleapis.com/roboflow-tests-assets/test-images/dog.jpeg"
+)
+
+
+@pytest.fixture()
+def roboflow_api_key() -> str:
+    return os.environ["ROBOFLOW_API_KEY"]
+
+
+@pytest.fixture(scope="function")
+def dog_image_numpy() -> np.ndarray:
+    _download_if_not_exists(file_path=DOG_IMAGE_PATH, url=DOG_IMAGE_URL)
+    image = cv2.imread(DOG_IMAGE_PATH)
+    assert image is not None, "Could not load test image"
+    return image
+
+
+def _download_if_not_exists(file_path: str, url: str, lock_timeout: int = 180) -> None:
+    os.makedirs(os.path.dirname(file_path), exist_ok=True)
+    lock_path = f"{file_path}.lock"
+    with FileLock(lock_file=lock_path, timeout=lock_timeout):
+        if os.path.exists(file_path):
+            return None
+        with requests.get(url, stream=True) as response:
+            response.raise_for_status()
+            with open(file_path, "wb") as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:
+                        f.write(chunk)

inference_experimental/tests/e2e_platform_tests/conftest.py

@@ -99,7 +99,7 @@ def _test_clip_wrapper_vs_baseline_for_image_embeddings(
 
     # then
     similarity = torch.nn.functional.cosine_similarity(
-        baseline_embeddings, wrapper_embeddings
+        baseline_embeddings.cpu(), wrapper_embeddings.cpu()
     )
     assert similarity.item() > 0.99
 

inference_experimental/tests/integration_tests/e2e/test_clip_e2e.py renamed to inference_experimental/tests/e2e_platform_tests/test_clip_e2e.py

@@ -0,0 +1,56 @@
+import numpy as np
+import pytest
+
+from inference_exp import AutoModel
+
+
+@pytest.mark.e2e_model_inference
+@pytest.mark.gpu_only
+def test_depth_anything_v2_small(roboflow_api_key: str, dog_image_numpy: np.ndarray) -> None:
+    # given
+    model = AutoModel.from_pretrained("depth-anything-v2/small", api_key=roboflow_api_key)
+
+    # when
+    results = model(dog_image_numpy)
+
+    # then
+    assert results[0].cpu().shape == (1280, 720)
+
+
+@pytest.mark.e2e_model_inference
+@pytest.mark.gpu_only
+def test_depth_anything_v2_small_via_alias(roboflow_api_key: str, dog_image_numpy: np.ndarray) -> None:
+    # given
+    model = AutoModel.from_pretrained("depth-anything-v2", api_key=roboflow_api_key)
+
+    # when
+    results = model(dog_image_numpy)
+
+    # then
+    assert results[0].cpu().shape == (1280, 720)
+
+
+@pytest.mark.e2e_model_inference
+@pytest.mark.gpu_only
+def test_depth_anything_v2_base(roboflow_api_key: str, dog_image_numpy: np.ndarray) -> None:
+    # given
+    model = AutoModel.from_pretrained("depth-anything-v2/base", api_key=roboflow_api_key)
+
+    # when
+    results = model(dog_image_numpy)
+
+    # then
+    assert results[0].cpu().shape == (1280, 720)
+
+
+@pytest.mark.e2e_model_inference
+@pytest.mark.gpu_only
+def test_depth_anything_v2_large(roboflow_api_key: str, dog_image_numpy: np.ndarray) -> None:
+    # given
+    model = AutoModel.from_pretrained("depth-anything-v2/large", api_key=roboflow_api_key)
+
+    # when
+    results = model(dog_image_numpy)
+
+    # then
+    assert results[0].cpu().shape == (1280, 720)

inference_experimental/tests/e2e_platform_tests/test_depth_anything_v2.py

@@ -16,10 +16,27 @@ def test_florence2_base_model(dog_image_numpy: np.ndarray):
     assert isinstance(captions, list)
     assert len(captions) == 1
     assert isinstance(captions[0], str)
-    assert captions[0] == "A man carrying a dog on his back."
+    assert len(captions[0]) > 0
 
 
 @pytest.mark.e2e_model_inference
+@pytest.mark.gpu_only
+def test_florence2_base_model(dog_image_numpy: np.ndarray):
+    # GIVEN
+    model = AutoModel.from_pretrained("florence-2-large")
+
+    # WHEN
+    captions = model.caption_image(dog_image_numpy)
+
+    # THEN
+    assert isinstance(captions, list)
+    assert len(captions) == 1
+    assert isinstance(captions[0], str)
+    assert len(captions[0]) > 0
+
+
+@pytest.mark.e2e_model_inference
+@pytest.mark.gpu_only
 def test_florence2_lora_model(
     dog_image_numpy: np.ndarray, dog_image_torch: torch.Tensor
 ):
@@ -33,4 +50,4 @@ def test_florence2_lora_model(
     assert isinstance(captions, list)
     assert len(captions) == 1
     assert isinstance(captions[0], str)
-    assert captions[0] == "Disease"
+    assert len(captions[0]) > 0