Feature/ecmwf ci

.github/ci-config.yml

@@ -0,0 +1,8 @@
+dependency_cmake_options: |
+  ecmwf/ecflow: "-DENABLE_ALL_TESTS=ON -DBOOST_ROOT=${BOOST_ROOT_DIR} -DBOOST_INCLUDEDIR=${BOOST_INCLUDE_DIR} -DBOOST_LIBRARYDIR=${BOOST_LIB_DIR} -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} -DENABLE_STATIC_BOOST_LIBS=OFF"
+dependencies: |
+  ecmwf/ecbuild
+  ecmwf/ecflow
+dependency_branch: develop
+parallelism_factor: 8
+self_build: false # Only for python packages

.github/ci-hpc-config.yml

@@ -0,0 +1,12 @@
+build:
+  modules:
+    - boost/1.84.0
+    - ninja
+    - python3/3.11.8-01
+    - qt
+    - cmake/new
+  dependencies:
+    - ecmwf/ecbuild@develop
+    - ecmwf/ecflow@develop
+  parallel: 64
+  pytest_cmd: pytest -rx

.github/workflows/ci.yml

@@ -1,75 +1,40 @@
-name: ci-pyflow
+name: ci
 
-# Controls when the workflow will run
 on:
+  # Trigger the workflow on push to master or develop, except tag creation
   push:
-    branches: [ "master", "develop" ]
-  pull_request:
-    branches: [ "master", "develop" ]
+    branches:
+      - 'master'
+      - 'develop'
+    tags-ignore:
+      - '**'
 
-jobs:
-  qa:
-    name: qa
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v4
-      - run: pip install black flake8 isort
-      - run: isort --check .
-      - run: black --check .
-      - run: flake8 .
+  # Trigger the workflow on pull request
+  pull_request: ~
+
+  # Trigger the workflow manually
+  workflow_dispatch: ~
 
-  docs:
-    name: docs
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: mamba-org/setup-micromamba@v1
-        with:
-         environment-name: pyflow-docs
-         environment-file: docs/environment.yml
-      - name: Install pyflow and docs
-        shell: bash -l {0}  # required to load conda
-        run: |
-          python -m pip install .
-          cd docs
-          make html
+  # Trigger after public PR approved for CI
+  pull_request_target:
+    types: [labeled]
 
-  setup:
-    name: setup
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: python -m pip install -r requirements.txt
-      - run: python -m pip install .
-
-  test:
-    name: pytest
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: mamba-org/setup-micromamba@v1
-        with:
-         environment-name: pyflow-test
-         environment-file: environment.yml
-      - name: Install pyflow and test
-        shell: bash -l {0}  # required to load conda
-        run: |
-          python -m pip install .
-          pytest
+jobs:
+  # Run CI including downstream packages on self-hosted runners
+  downstream-ci:
+    name: downstream-ci
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@feature/pyflow
+    with:
+      pyflow: ecmwf/pyflow@${{ github.event.pull_request.head.sha || github.sha }}
+      python_qa: true
+    secrets: inherit
 
-  notify:
-    name: notify
-    runs-on: ubuntu-20.04
-    needs:
-    - qa
-    - setup
-    - test
-    if: always() && inputs.notify_teams && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop')
-    steps:
-    - name: Notify Teams
-      uses: ecmwf-actions/notify-teams@v1
-      with:
-        incoming_webhook: ${{ secrets.incoming_webhook }}
-        needs_context: ${{ toJSON(needs) }}
+  # Build downstream packages on HPC
+  downstream-ci-hpc:
+    name: downstream-ci-hpc
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@feature/pyflow
+    with:
+      pyflow: ecmwf/pyflow@${{ github.event.pull_request.head.sha || github.sha }}
+    secrets: inherit

.github/workflows/ci_legacy.yml

@@ -0,0 +1,75 @@
+name: ci-pyflow
+
+# Controls when the workflow will run
+on:
+  push:
+    branches: [ "master", "develop" ]
+  pull_request:
+    branches: [ "master", "develop" ]
+
+jobs:
+  qa:
+    name: qa
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v4
+      - run: pip install black flake8 isort
+      - run: isort --check .
+      - run: black --check .
+      - run: flake8 .
+
+  docs:
+    name: docs
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: mamba-org/setup-micromamba@v1
+        with:
+         environment-name: pyflow-docs
+         environment-file: docs/environment.yml
+      - name: Install pyflow and docs
+        shell: bash -l {0}  # required to load conda
+        run: |
+          python -m pip install .
+          cd docs
+          make html
+
+  setup:
+    name: setup
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: python -m pip install -r requirements.txt
+      - run: python -m pip install .
+
+  test:
+    name: pytest
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: mamba-org/setup-micromamba@v1
+        with:
+         environment-name: pyflow-test
+         environment-file: environment.yml
+      - name: Install pyflow and test
+        shell: bash -l {0}  # required to load conda
+        run: |
+          python -m pip install .
+          pytest
+
+  notify:
+    name: notify
+    runs-on: ubuntu-20.04
+    needs:
+    - qa
+    - setup
+    - test
+    if: always() && inputs.notify_teams && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop')
+    steps:
+    - name: Notify Teams
+      uses: ecmwf-actions/notify-teams@v1
+      with:
+        incoming_webhook: ${{ secrets.incoming_webhook }}
+        needs_context: ${{ toJSON(needs) }}

.github/workflows/label-public-pr.yml

@@ -0,0 +1,10 @@
+# Manage labels of pull requests that originate from forks
+name: label-public-pr
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+
+jobs:
+  label:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2

environment.yml

@@ -11,6 +11,8 @@ dependencies:
   - requests
   - pytest
   - pygraphviz
+  - pip:
+    - findlibs
 
 variables:
   QT_MAC_WANTS_LAYER: 1

pyflow/importer.py

@@ -1,6 +1,13 @@
 import os
 import sys
 
+try:
+    import ecmwflibs as findlibs
+except ImportError:
+    import findlibs
+
+library_path = findlibs.find("eccodes")
+
 try:
     import ecflow
 except ImportError:
@@ -9,14 +16,17 @@
     for env_var in ("ECFLOW_DIR", "ecflow_DIR"):
         try:
             lib_dir = os.path.join(os.environ[env_var], "lib")
+            print("DF: Looking for lib_dir: {}".format(lib_dir))
         except KeyError:
             continue
 
         for minor_version in reversed(range(6, sys.version_info.minor + 1)):
             python_dir = os.path.join(
                 lib_dir, "python3.{}".format(minor_version), "site-packages"
             )
+            print("DF: Looking for python_dir: {}".format(python_dir))
             if os.path.exists(python_dir):
+                print("DF: Found python_dir: {}".format(python_dir))
                 sys.path.insert(0, python_dir)
                 import ecflow  # noqa: F401
 

requirements.txt

@@ -2,3 +2,4 @@ jinja2
 requests
 pytest
 graphviz
+findlibs