PR #1615: GitHub CI cleanups

Author: reidpr

.github/workflows/main.yml

@@ -14,23 +14,22 @@ jobs:
     strategy:
       fail-fast: false  # Actions seems kind of flaky lately
       matrix:
-        builder: [none, docker, ch-image]
+        builder:
+          - ch-image
         pack_fmt: [squash-mount, tar-unpack, squash-unpack]
         keep_sudo:   # if false, remove self from sudoers post-install/setup
           - false
-        cache: [enabled, disabled]
-        exclude:
+        cache:
+          - enabled
+        include:
           - builder: none
-            cache: enabled
-          - builder: docker
-            cache: enabled
-          - builder: ch-image
+            pack_fmt: squash-mount
+            keep_sudo: false
             cache: disabled
-        include:
           - builder: docker
-            pack_fmt: squash-mount
+            pack_fmt: tar-unpack
             keep_sudo: true
-            cache: disabled
+            cache: enabled
           - builder: ch-image
             pack_fmt: squash-mount
             keep_sudo: false
@@ -65,8 +64,8 @@ jobs:
           echo "PATH=$path_new" >> $GITHUB_ENV
           # Set sudo umask to something quite restrictive. The default is
           # 0022, but we had a "make install" bug (issue #947) that was
-          # tickled by 0027, which is a better setting. For reasons I don't
-          # understand, this only affects sudo, but since that's what we want,
+          # tickled by 0027, which is a better setting. For reasons I don’t
+          # understand, this only affects sudo, but since that’s what we want,
           # I left it. Note there are a few dependency installs below that
           # have similar permissions bugs; these relax the umask on a
           # case-by-case basis.
@@ -104,7 +103,7 @@ jobs:
 
       - name: install/configure dependencies, all
         run: |
-          # configure doesn't tell us about these.
+          # configure doesn’t tell us about these.
           sudo apt-get install attr pigz pv
           # configure does tell us about these.
           sudo apt-get install bats squashfs-tools
@@ -162,7 +161,6 @@ jobs:
           sudo ldconfig
           # Unset setuid on all fusermount3(1) on the system.
           sudo chmod -v u-s /usr/bin/fusermount*
-          sudo find / -name 'fusermount*' -ls
           ls -lh $(command -v fusermount3)
           [[ ! -u $(command -v fusermount3) ]]
 
@@ -172,7 +170,7 @@ jobs:
         run: |
           sudo apt-get install -y musl-tools
           # At present we use argp to parse command line arguments. This is
-          # not portable; it's a glibc extension (see #1260). Thus, install a
+          # not portable; it’s a glibc extension (see #1260). Thus, install a
           # standalone argp.
           cd /usr/local/src
           git clone --depth=1 https://github.com/ericonr/argp-standalone.git
@@ -211,7 +209,9 @@ jobs:
           sudo usermod --add-subgids 10000-65536 $USER
 
       - name: disable bundled lark, ch-image
-        if: ${{ matrix.builder == 'ch-image' && matrix.cache == 'disabled' }}
+        if: ${{    matrix.builder == 'ch-image'
+                && matrix.pack_fmt == 'squash-mount'
+                && matrix.cache == 'disabled' }}
         run: |
           set -x
           # no install, disable
@@ -255,7 +255,7 @@ jobs:
           # workflow also use glibc.
           if [[ $CH_TEST_PACK_FMT == tar-unpack ]]; then
               export CC=musl-gcc
-              # GNU ldd(1) doesn't work on musl binaries.
+              # GNU ldd(1) doesn’t work on musl binaries.
               LDD='/lib/ld-musl-x86_64.so.1 --list'
           else
               LDD=ldd
@@ -330,23 +330,10 @@ jobs:
                   -v ~/registry-etc:/etc/docker/registry registry:2
           docker ps -a
 
-      - name: configure sudo to user root, group non-root
-        if: ${{ matrix.keep_sudo }}
-        run: |
-          sudo sed -Ei 's/=\(ALL\)/=(ALL:ALL)/g' /etc/sudoers.d/runner
-          sudo cat /etc/sudoers.d/runner
-
-      - name: remove sudo
-        if: ${{ ! matrix.keep_sudo }}
-        run: |
-          sudo rm /etc/sudoers.d/runner
-          ! sudo echo hello
-
       - name: build/install from tarball
-        if: ${{ matrix.builder == 'docker' && matrix.keep_sudo }}
         run: |
           # Create and unpack tarball. The wildcard saves us having to put the
-          # version in a variable. This assumes there isn't already a tarball
+          # version in a variable. This assumes there isn’t already a tarball
           # or unpacked directory in $ch_prefix, which is true on the clean
           # VMs GitHub gives us. Note that cd fails if it gets more than one
           # argument, which helps, but this is probably kind of brittle.
@@ -380,29 +367,38 @@ jobs:
           bin/ch-run --version
           $ch_prefix/from-tarball/bin/ch-run --version
 
-      - name: run test suite (Git WD, quick)
-        if: ${{ matrix.builder == 'docker' && ! matrix.keep_sudo }}
+      - name: configure sudo to user root, group non-root
+        if: ${{ matrix.keep_sudo }}
         run: |
-          bin/ch-test --scope=quick all
+          sudo sed -Ei 's/=\(ALL\)/=(ALL:ALL)/g' /etc/sudoers.d/runner
+          sudo cat /etc/sudoers.d/runner
+
+      - name: remove sudo
+        if: ${{ ! matrix.keep_sudo }}
+        run: |
+          sudo rm /etc/sudoers.d/runner
+          ! sudo echo hello
 
       - name: run test suite (Git WD, standard)
         run: |
           bin/ch-test all
 
       - name: run test suite (installed from Git WD, standard)
-        if: ${{    ( matrix.builder == 'docker' && ! matrix.keep_sudo )
-                || (    matrix.builder == 'ch-image'
-                     && matrix.cache == 'disabled' ) }}
+        if: ${{    matrix.builder == 'ch-image'
+                && matrix.pack_fmt == 'squash-mount'
+                && matrix.cache == 'enabled' }}
         run: |
           $ch_prefix/from-git/bin/ch-test all
 
       - name: run test suite (installed from tarball, standard)
-        if: ${{ matrix.builder == 'docker' && matrix.keep_sudo }}
+        if: ${{    matrix.builder == 'ch-image'
+                && matrix.pack_fmt == 'squash-mount'
+                && matrix.cache == 'enabled' }}
         run: |
           $ch_prefix/from-tarball/bin/ch-test all
 
-      - name: make uninstall
-        if: ${{ matrix.builder == 'docker' && matrix.keep_sudo }}
+      - name: uninstall from tarball
+        if: ${{ matrix.keep_sudo }}
         run: |
           cd $ch_prefix/charliecloud-*
           sudo make uninstall
@@ -419,7 +415,7 @@ jobs:
           EOF
 
       - name: rebuild with most things --disable’d
-        if: ${{ matrix.builder == 'docker' && ! matrix.keep_sudo }}
+        if: ${{ matrix.builder == 'docker' }}
         run: |
           make distclean
           ./configure --prefix=/doesnotexist \
@@ -438,31 +434,29 @@ jobs:
           bin/ch-run --version
 
       - name: run test suite (Git WD, standard)
-        if: ${{ matrix.builder == 'docker' && ! matrix.keep_sudo }}
+        if: ${{ matrix.builder == 'docker' }}
         run: |
           bin/ch-test all
 
       - name: remove non-essential dependencies
-        if: ${{ matrix.builder == 'docker' && matrix.keep_sudo && matrix.pack_fmt == 'tar-unpack' }}
+        if: ${{ matrix.builder == 'docker' }}
         run: |
           set -x
           # This breaks lots of dependencies unrelated to our build but YOLO.
           sudo dpkg --remove --force-depends \
                     pigz \
                     pv \
                     python3-requests \
-                    squashfs-tools \
+                    squashfs-tools
           sudo pip3 uninstall -y sphinx sphinx-rtd-theme
           if [[ ${{ matrix.pack_fmt }} = squash-mount ]]; then
-              ( cd /usr/local/src/squashfuse && make uninstall )
+              ( cd /usr/local/src/squashfuse && sudo make uninstall )
           fi
           ! python3 -c 'import requests'
           ! python3 -c 'import lark'
-          test -e bin/ch-image
-          bin/ch-test -f test/build/10_sanity.bats  # issue #806
 
       - name: rebuild
-        if: ${{ matrix.builder == 'docker' && matrix.keep_sudo && matrix.pack_fmt == 'tar-unpack' }}
+        if: ${{ matrix.builder == 'docker' }}
         run: |
           make distclean
           ./configure --prefix=/doesnotexist
@@ -478,7 +472,7 @@ jobs:
           bin/ch-run --version
 
       - name: run test suite (Git WD, standard)
-        if: ${{ matrix.builder == 'docker' && matrix.keep_sudo && matrix.pack_fmt == 'tar-unpack' }}
+        if: ${{ matrix.builder == 'docker' }}
         run: |
           bin/ch-test all
 

examples/Dockerfile.centos_7ch

@@ -1,4 +1,4 @@
-# ch-test-scope: standard
+# ch-test-scope: full
 FROM centos:7
 
 # This image has two purposes: (1) demonstrate we can build a CentOS 7 image

lib/charliecloud.py

@@ -19,19 +19,15 @@
 import time
 import traceback
 
-import filesystem as fs
-import registry as rg
-import version
-
-# Compatibility link. Sometimes we load pickled data from when Path was
-# defined in this file. This alias lets us still load such pickles.
-Path = fs.Path
 
+# List of dependency problems. This variable needs to be created before we
+# import any other Charliecloud stuff to avoid #806.
+depfails = []
 
-## Hairy imports ##
 
-# List of dependency problems.
-depfails = []
+import filesystem as fs
+import registry as rg
+import version
 
 
 ## Enums ##
@@ -104,6 +100,10 @@ class Download_Mode(enum.Enum):
 
 ## Globals ##
 
+# Compatibility link. Sometimes we load pickled data from when Path was
+# defined in this file. This alias lets us still load such pickles.
+Path = fs.Path
+
 # Active architecture (both using registry vocabulary)
 arch = None       # requested by user
 arch_host = None  # of host

misc/loc

@@ -89,6 +89,7 @@ Dockerfile
     filename Dockerfile.nvidia
     filename Dockerfile.ocimanifest
     filename Dockerfile.openmpi
+    filename Dockerfile.quick
     3rd_gen_scale 2.00
     end_of_line_continuation \\$
 m4

test/Dockerfile.quick

@@ -0,0 +1,5 @@
+# Minimal test image to exercise a Dockerfile build in quick scope.
+# ch-test-scope: quick
+
+FROM alpine:3.17
+RUN apk add bc

test/Makefile.am

@@ -4,6 +4,7 @@ testdir = $(pkglibexecdir)/test
 testfiles = \
 .dockerignore \
 Dockerfile.argenv \
+Dockerfile.quick \
 approved-trailing-whitespace \
 bucache/a.df \
 bucache/a-fail.df \

test/build/10_sanity.bats

@@ -128,7 +128,7 @@ load ../common
 }
 
 @test 'proxy variables' {
-    scope quick
+    scope standard
     # Proxy variables are a mess on UNIX. There are a lot them, and different
     # programs use them inconsistently. This test is based on the assumption
     # that if one of the proxy variables are set, then they all should be, in

test/build/50_dockerfile.bats

@@ -564,7 +564,7 @@ EOF
 
    # test that it works with python3
    run build_ -t tmpimg -f - . <<'EOF'
-FROM centos_7ch
+FROM almalinux_8ch
 SHELL ["/usr/bin/python3", "-c"]
 RUN print ("hello")
 EOF

test/build/55_cache.bats

@@ -246,9 +246,9 @@ EOF
 *  (HEAD -> root) ROOT
 EOF
 )
-    ch-image --rebuild build -t c -f bucache/c.df .
     # avoid race condition
     sleep 1
+    ch-image --rebuild build -t c -f bucache/c.df .
     run ch-image build-cache --tree
     [[ $status -eq 0 ]]
     diff -u <(echo "$blessed_out") <(echo "$output" | treeonly)

test/force-auto.py.in

@@ -122,7 +122,8 @@ class Test(abc.ABC):
          print(f"\n# skip: {self}: --force=%s excluded" % self.force)
          return
       # scope
-      if (self.scope == Scope.STANDARD or self.run == Run.NEEDED):
+      if (    (self.scope == Scope.STANDARD or self.run == Run.NEEDED)
+          and self.force != "fakeroot"):
          scope = "standard"
       else:
          scope = "full"