LLNL · dreamer2368 · May 3, 2023 · Jan 31, 2023 · Feb 5, 2023 · Feb 5, 2023
diff --git a/.github/workflows/Dockerfile b/.github/workflows/Dockerfile
@@ -0,0 +1,148 @@
+FROM ubuntu:22.04
+
+ENV ENVDIR=env
+
+# install sudo
+RUN apt-get -yq update && apt-get -yq install sudo
+
+WORKDIR /$ENVDIR
+
+# install packages
+RUN sudo apt-get install -yq git
+RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake
+RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-serial-dev
+RUN sudo apt-get install -yq vim
+RUN sudo apt-get install -yq git-lfs
+RUN sudo apt-get install -yq valgrind
+RUN sudo apt-get install -yq wget
+
+RUN sudo apt-get clean -q
+
+# download dependencies
+ENV LIB_DIR=/$ENVDIR/dependencies
+WORKDIR $LIB_DIR
+
+#RUN wget -O mfem-4.5.tar.gz https://github.com/mfem/mfem/archive/refs/tags/v4.5.tar.gz
+RUN wget -O hypre-2.20.0.tar.gz https://github.com/hypre-space/hypre/archive/refs/tags/v2.20.0.tar.gz
+
+# Instead of the original parmetis link (which is often unavailable), use the link to librom master branch:
+# RUN wget -O parmetis-4.0.3.tar.gz http://glaros.dtc.umn.edu/gkhome/fetch/sw/parmetis/parmetis-4.0.3.tar.gz
+RUN wget -O parmetis-4.0.3.tar.gz https://github.com/LLNL/libROM/raw/master/dependencies/parmetis-4.0.3.tar.gz
+
+RUN wget -O gslib-1.0.7.tar.gz https://github.com/gslib/gslib/archive/v1.0.7.tar.gz
+#RUN wget -O glvis-4.2.tar.gz https://github.com/GLVis/glvis/archive/refs/tags/v4.2.tar.gz
+#RUN wget -O metis-4.0.3.tar.gz https://github.com/mfem/tpls/raw/gh-pages/metis-4.0.3.tar.gz
+
+
+ENV CFLAGS="-fPIC"
+ENV CPPFLAGS="-fPIC"
+ENV CXXFLAGS="-fPIC"
+
+# install hypre
+RUN tar -zxvf hypre-2.20.0.tar.gz
+RUN mv hypre-2.20.0 hypre
+WORKDIR ./hypre/src/
+RUN ./configure --disable-fortran
+RUN make -j
+WORKDIR $LIB_DIR
+
+# install gslib
+ENV MG=YES
+RUN tar -zxvf gslib-1.0.7.tar.gz
+RUN mv gslib-1.0.7 gslib
+WORKDIR ./gslib
+RUN make CC=mpicc -j
+WORKDIR $LIB_DIR
+
+## install metis
+#RUN tar -zxvf metis-4.0.3.tar.gz
+#WORKDIR ./metis-4.0.3
+#RUN make OPTFLAGS=-Wno-error=implicit-function-declaration
+#WORKDIR $LIB_DIR
+#RUN ln -s metis-4.0.3 metis-4.0
+
+# install parmetis
+RUN tar -zxvf parmetis-4.0.3.tar.gz
+WORKDIR ./parmetis-4.0.3
+RUN make config
+RUN make
+
+# These environment variables are for mfem build.
+# Need different values for libROM build.
+ENV METIS_DIR=${LIB_DIR}/parmetis-4.0.3
+ENV METIS_OPT=-I${METIS_DIR}/metis/include
+WORKDIR ${METIS_DIR}/build
+# Currently docker cannot save command results to environment variables.
+# These variables are for libROM build.
+ENV MACHINE_ARCH=Linux-x86_64
+RUN ln -s $MACHINE_ARCH lib
+WORKDIR ${METIS_DIR}/build/lib/libparmetis
+RUN ln -s ./ lib
+RUN ln -s ${METIS_DIR}/metis metis
+WORKDIR ${METIS_DIR}/build/lib/libmetis
+RUN ln -s ./ lib
+ENV METIS_LIB="-L${METIS_DIR}/build/lib/libparmetis -lparmetis -L${METIS_DIR}/build/lib/libmetis -lmetis"
+
+ENV CFLAGS=
+ENV CPPFLAGS=
+ENV CXXFLAGS=
+
+WORKDIR $LIB_DIR
+
+# install mfem
+RUN git clone https://github.com/mfem/mfem.git mfem_parallel
+WORKDIR ./mfem_parallel
+# Latest verified commit on May 2, 2023
+RUN git checkout e5231334e6a8175b4f404b877f590b73dee2dedc
+#RUN tar -zxvf mfem-4.5.tar.gz
+#RUN mv mfem-4.5 mfem_parallel
+#RUN git pull
+#RUN make serial -j 4
+RUN make parallel -j 4 STATIC=NO SHARED=YES MFEM_USE_MPI=YES MFEM_USE_GSLIB=${MG} MFEM_USE_METIS=YES MFEM_USE_METIS_5=YES METIS_DIR="$METIS_DIR" METIS_OPT="$METIS_OPT" METIS_LIB="$METIS_LIB"
+RUN ln -s ./ lib
+RUN ln -s ./ include
+WORKDIR $LIB_DIR
+RUN ln -s mfem_parallel mfem
+
+# install googletest
+WORKDIR $LIB_DIR
+RUN git clone https://github.com/google/googletest
+WORKDIR ./googletest
+# Last release that supports c++11
+RUN git checkout tags/release-1.12.1 -b v1.12.1
+WORKDIR ./build
+RUN cmake ..
+RUN make
+RUN sudo make install
+
+# clean up
+WORKDIR $LIB_DIR
+RUN rm *.tar.gz
+
+# cmake toolchain file for librom
+RUN echo 'set(CMAKE_C_COMPILER mpicc)\n\
+set(CMAKE_CXX_COMPILER mpicxx)\n\
+set(CMAKE_Fortran_COMPILER mpif90)\n\
+set(LIB_DIR /env/dependencies)\n\
+set(MFEM_DIR ${LIB_DIR}/mfem)\n\
+set(HYPRE_DIR ${LIB_DIR}/hypre/src/hypre)\n\
+set(PARMETIS_DIR ${LIB_DIR}/parmetis-4.0.3/build/lib/libparmetis)\n\
+set(METIS_DIR ${LIB_DIR}/parmetis-4.0.3/build/lib/libmetis)' > ./librom_env.cmake
+
+# flags for libROM cmake
+ENV TOOLCHAIN_FILE=$LIB_DIR/librom_env.cmake
+ENV BUILD_TYPE=Optimized
+ENV USE_MFEM=On
+ENV MFEM_USE_GSLIB=On
+ENV MFEM_DIR=$LIB_DIR/mfem
+ENV HYPRE_DIR=$LIB_DIR/hypre/src/hypre
+ENV PARMETIS_DIR=$LIB_DIR/parmetis-4.0.3/build/lib/libparmetis
+ENV METIS_DIR=$LIB_DIR/parmetis-4.0.3/build/lib/libmetis
+
+# create and switch to a user
+ENV USERNAME=test
+RUN echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+RUN useradd --no-log-init -u 1001 --create-home --shell /bin/bash $USERNAME
+RUN adduser $USERNAME sudo
+USER $USERNAME
+WORKDIR /home/$USERNAME
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,35 +1,32 @@
 name: CI
-on: 
+on:
+  workflow_dispatch: {}
   pull_request:
     types: [opened, labeled, synchronize]
     branches:
       - master
 jobs:
   linux:
     runs-on: ubuntu-latest
+    container:
+      image: dreamer2368/librom_env:latest
+      options: --user 1001 --privileged
+      volumes:
+        - /mnt:/mnt
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
-        with: 
+        with:
           swap-size-gb: 10
-
-      - name: Install Linux dependencies
-        run: |
-            sudo apt update
-            sudo apt-get install libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-serial-dev
       - name: Check out libROM
-        uses: actions/checkout@v2
-      - uses: ./.github/workflows/checkout_repo
+        uses: actions/checkout@v3
       - name: Build libROM
         run: |
             mkdir ${GITHUB_WORKSPACE}/build
-            export CC=mpicc
-            export CXX=mpicxx
-            scripts/setup.sh
             cd ${GITHUB_WORKSPACE}/build
-            cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_MFEM=On ..
+            cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Debug -DUSE_MFEM=${USE_MFEM} -DMFEM_USE_GSLIB=${MFEM_USE_GSLIB}
             make
-            cmake -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=On ..
+            cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=${USE_MFEM} -DMFEM_USE_GSLIB=${MFEM_USE_GSLIB}
             make
       - name: Build baseline libROM
         if: ${{ github.event.label.name == 'LGTM' || contains(github.event.pull_request.labels.*.name, 'LGTM') }}
@@ -38,35 +35,32 @@ jobs:
             git clone https://github.com/LLNL/libROM.git
             cd libROM
             mkdir build
-            export CC=mpicc
-            export CXX=mpicxx
-            scripts/setup.sh
             cd build
-            cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_MFEM=On ..
+            cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Debug -DUSE_MFEM=${USE_MFEM} -DMFEM_USE_GSLIB=${MFEM_USE_GSLIB}
             make
-            cmake -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=On ..
+            cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=${USE_MFEM} -DMFEM_USE_GSLIB=${MFEM_USE_GSLIB}
             make
       - uses: ./.github/workflows/run_tests
-  mac:
-    runs-on: macos-latest
-    steps:
-      - name: Install Mac dependencies
-        run: |
-            brew install open-mpi
-            brew install openblas
-            brew install lapack
-            brew install scalapack
-            brew install hdf5
-      - name: Check out libROM
-        uses: actions/checkout@v2
-      - uses: ./.github/workflows/checkout_repo
-      - name: Build libROM
-        run: |
-            export FC=/usr/local/bin/gfortran-10
-            mkdir ${GITHUB_WORKSPACE}/build
-            cd ${GITHUB_WORKSPACE}/build
-            cmake -DCMAKE_BUILD_TYPE=Debug ..
-            make
-            cmake -DCMAKE_BUILD_TYPE=Optimized ..
-            make
-      - uses: ./.github/workflows/run_tests
+  # mac:
+  #   runs-on: macos-latest
+  #   steps:
+  #     - name: Install Mac dependencies
+  #       run: |
+  #           brew install open-mpi
+  #           brew install openblas
+  #           brew install lapack
+  #           brew install scalapack
+  #           brew install hdf5
+  #     - name: Check out libROM
+  #       uses: actions/checkout@v2
+  #     - uses: ./.github/workflows/checkout_repo
+  #     - name: Build libROM
+  #       run: |
+  #           export FC=/usr/local/bin/gfortran-10
+  #           mkdir ${GITHUB_WORKSPACE}/build
+  #           cd ${GITHUB_WORKSPACE}/build
+  #           cmake -DCMAKE_BUILD_TYPE=Debug ..
+  #           make
+  #           cmake -DCMAKE_BUILD_TYPE=Optimized ..
+  #           make
+  #     - uses: ./.github/workflows/run_tests
diff --git a/.github/workflows/run_tests/action.yml b/.github/workflows/run_tests/action.yml
@@ -13,19 +13,18 @@ runs:
           ./tests/test_S_OPT
           mpirun -n 4 --oversubscribe tests/test_S_OPT
           ./tests/test_IncrementalSVD
-          ./tests/test_RandomizedSVD
-          mpirun -n 3 --oversubscribe tests/test_RandomizedSVD
           ./tests/test_DMD
           mpirun -n 3 --oversubscribe tests/test_DMD
           ./tests/test_GreedyCustomSampler
           mpirun -n 3 --oversubscribe tests/test_GreedyCustomSampler
-
+# ./tests/test_RandomizedSVD
+# mpirun -n 3 --oversubscribe tests/test_RandomizedSVD
       shell: bash
-      
+
     - name: Run regression tests
       if: ${{ github.event.label.name == 'LGTM' || contains(github.event.pull_request.labels.*.name, 'LGTM') }}
       run: |
           cd ${GITHUB_WORKSPACE}
           ./regression_tests/run_regression_tests.sh
-         
+
       shell: bash