diff --git a/.github/ci-config.yml b/.github/ci-config.yml new file mode 100644 index 0000000..94011ab --- /dev/null +++ b/.github/ci-config.yml @@ -0,0 +1,8 @@ +dependency_cmake_options: | + ecmwf/ecflow: "-DENABLE_ALL_TESTS=ON -DBOOST_ROOT=${BOOST_ROOT_DIR} -DBOOST_INCLUDEDIR=${BOOST_INCLUDE_DIR} -DBOOST_LIBRARYDIR=${BOOST_LIB_DIR} -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} -DENABLE_STATIC_BOOST_LIBS=OFF" +dependencies: | + ecmwf/ecbuild + ecmwf/ecflow +dependency_branch: develop +parallelism_factor: 8 +self_build: false # Only for python packages diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml new file mode 100644 index 0000000..5e6a074 --- /dev/null +++ b/.github/ci-hpc-config.yml @@ -0,0 +1,12 @@ +build: + modules: + - boost/1.84.0 + - ninja + - python3/3.11.8-01 + - qt + - cmake/new + dependencies: + - ecmwf/ecbuild@develop + - ecmwf/ecflow@develop + parallel: 64 + pytest_cmd: pytest -rx diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31b2f10..a3e4bc9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,75 +1,40 @@ -name: ci-pyflow +name: ci -# Controls when the workflow will run on: + # Trigger the workflow on push to master or develop, except tag creation push: - branches: [ "master", "develop" ] - pull_request: - branches: [ "master", "develop" ] + branches: + - 'master' + - 'develop' + tags-ignore: + - '**' -jobs: - qa: - name: qa - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v4 - - run: pip install black flake8 isort - - run: isort --check . - - run: black --check . - - run: flake8 . + # Trigger the workflow on pull request + pull_request: ~ + + # Trigger the workflow manually + workflow_dispatch: ~ - docs: - name: docs - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: mamba-org/setup-micromamba@v1 - with: - environment-name: pyflow-docs - environment-file: docs/environment.yml - - name: Install pyflow and docs - shell: bash -l {0} # required to load conda - run: | - python -m pip install . - cd docs - make html + # Trigger after public PR approved for CI + pull_request_target: + types: [labeled] - setup: - name: setup - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: python -m pip install -r requirements.txt - - run: python -m pip install . - - test: - name: pytest - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: mamba-org/setup-micromamba@v1 - with: - environment-name: pyflow-test - environment-file: environment.yml - - name: Install pyflow and test - shell: bash -l {0} # required to load conda - run: | - python -m pip install . - pytest +jobs: + # Run CI including downstream packages on self-hosted runners + downstream-ci: + name: downstream-ci + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@feature/pyflow + with: + pyflow: ecmwf/pyflow@${{ github.event.pull_request.head.sha || github.sha }} + python_qa: true + secrets: inherit - notify: - name: notify - runs-on: ubuntu-20.04 - needs: - - qa - - setup - - test - if: always() && inputs.notify_teams && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop') - steps: - - name: Notify Teams - uses: ecmwf-actions/notify-teams@v1 - with: - incoming_webhook: ${{ secrets.incoming_webhook }} - needs_context: ${{ toJSON(needs) }} + # Build downstream packages on HPC + downstream-ci-hpc: + name: downstream-ci-hpc + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@feature/pyflow + with: + pyflow: ecmwf/pyflow@${{ github.event.pull_request.head.sha || github.sha }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/ci_legacy.yml b/.github/workflows/ci_legacy.yml new file mode 100644 index 0000000..31b2f10 --- /dev/null +++ b/.github/workflows/ci_legacy.yml @@ -0,0 +1,75 @@ +name: ci-pyflow + +# Controls when the workflow will run +on: + push: + branches: [ "master", "develop" ] + pull_request: + branches: [ "master", "develop" ] + +jobs: + qa: + name: qa + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v4 + - run: pip install black flake8 isort + - run: isort --check . + - run: black --check . + - run: flake8 . + + docs: + name: docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: mamba-org/setup-micromamba@v1 + with: + environment-name: pyflow-docs + environment-file: docs/environment.yml + - name: Install pyflow and docs + shell: bash -l {0} # required to load conda + run: | + python -m pip install . + cd docs + make html + + setup: + name: setup + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: python -m pip install -r requirements.txt + - run: python -m pip install . + + test: + name: pytest + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: mamba-org/setup-micromamba@v1 + with: + environment-name: pyflow-test + environment-file: environment.yml + - name: Install pyflow and test + shell: bash -l {0} # required to load conda + run: | + python -m pip install . + pytest + + notify: + name: notify + runs-on: ubuntu-20.04 + needs: + - qa + - setup + - test + if: always() && inputs.notify_teams && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop') + steps: + - name: Notify Teams + uses: ecmwf-actions/notify-teams@v1 + with: + incoming_webhook: ${{ secrets.incoming_webhook }} + needs_context: ${{ toJSON(needs) }} diff --git a/.github/workflows/label-public-pr.yml b/.github/workflows/label-public-pr.yml new file mode 100644 index 0000000..52f3b87 --- /dev/null +++ b/.github/workflows/label-public-pr.yml @@ -0,0 +1,10 @@ +# Manage labels of pull requests that originate from forks +name: label-public-pr + +on: + pull_request_target: + types: [opened, synchronize] + +jobs: + label: + uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2 \ No newline at end of file diff --git a/environment.yml b/environment.yml index d4a587c..634d0f0 100644 --- a/environment.yml +++ b/environment.yml @@ -11,6 +11,8 @@ dependencies: - requests - pytest - pygraphviz + - pip: + - findlibs variables: QT_MAC_WANTS_LAYER: 1 diff --git a/pyflow/importer.py b/pyflow/importer.py index 870c7b8..09620cf 100644 --- a/pyflow/importer.py +++ b/pyflow/importer.py @@ -1,6 +1,13 @@ import os import sys +try: + import ecmwflibs as findlibs +except ImportError: + import findlibs + +library_path = findlibs.find("eccodes") + try: import ecflow except ImportError: @@ -9,6 +16,7 @@ for env_var in ("ECFLOW_DIR", "ecflow_DIR"): try: lib_dir = os.path.join(os.environ[env_var], "lib") + print("DF: Looking for lib_dir: {}".format(lib_dir)) except KeyError: continue @@ -16,7 +24,9 @@ python_dir = os.path.join( lib_dir, "python3.{}".format(minor_version), "site-packages" ) + print("DF: Looking for python_dir: {}".format(python_dir)) if os.path.exists(python_dir): + print("DF: Found python_dir: {}".format(python_dir)) sys.path.insert(0, python_dir) import ecflow # noqa: F401 diff --git a/requirements.txt b/requirements.txt index 6e214ff..cae7a23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ jinja2 requests pytest graphviz +findlibs