Skip to content

Add PCRE2 support #3432

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add PCRE2 support #3432

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 31 additions & 84 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,106 +15,55 @@ jobs:
arch:
- amd64
- arm64
- armel
- armhf
- i386
- mips
- mips64
- mips64el
- mips64r6
- mips64r6el
- mipsel
- mipsr6
- mipsr6el
- powerpc
- ppc64el
- riscv64
- s390x
include:
# TODO: arm/v7 arm/v6?
- arch: amd64
CC: x86_64-linux-gnu
platform: linux/amd64
- arch: arm64
CC: aarch64-linux-gnu
- arch: armel
CC: arm-linux-gnueabi
- arch: armhf
CC: arm-linux-gnueabihf
platform: linux/arm64
- arch: i386
CC: i686-linux-gnu
- arch: mips
CC: mips-linux-gnu
- arch: mips64
CC: mips64-linux-gnuabi64
- arch: mips64el
CC: mips64el-linux-gnuabi64
- arch: mips64r6
CC: mipsisa64r6-linux-gnuabi64
- arch: mips64r6el
CC: mipsisa64r6el-linux-gnuabi64
- arch: mipsel
CC: mipsel-linux-gnu
- arch: mipsr6
CC: mipsisa32r6-linux-gnu
- arch: mipsr6el
CC: mipsisa32r6el-linux-gnu
- arch: powerpc
CC: powerpc-linux-gnu
platform: linux/386
- arch: ppc64el
CC: powerpc64le-linux-gnu
platform: linux/ppc64le
- arch: riscv64
CC: riscv64-linux-gnu
platform: linux/riscv64
- arch: s390x
CC: s390x-linux-gnu
runs-on: ${{ matrix.arch == 'i386' && 'ubuntu-22.04' || 'ubuntu-24.04' }}
platform: linux/s390x
runs-on: ubuntu-24.04
env:
AR: ${{ matrix.CC }}-ar
CHOST: ${{ matrix.CC }}
CC: ${{ matrix.CC }}-gcc
CPP: ${{ matrix.CC }}-cpp
LDFLAGS: -s
SUFFIX: linux-${{ matrix.arch }}
BIN_NAME: jq-linux-${{ matrix.arch }}
steps:
- name: Clone repository
uses: actions/checkout@v5
with:
submodules: true
- name: Install packages
run: |
sudo apt-get update
sudo apt-get install -y automake autoconf libtool crossbuild-essential-${{ matrix.arch }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build
run: |
autoreconf -i
./configure \
--host=${{ matrix.CC }} \
--disable-docs \
--with-oniguruma=builtin \
--enable-static \
--enable-all-static \
CFLAGS="-O2 -pthread -fstack-protector-all"
make -j"$(nproc)"
file ./jq
cp ./jq jq-${{ env.SUFFIX }}
- name: Test
# Only run tests for amd64 matching the CI machine arch
if: ${{ matrix.arch == 'amd64' }}
run: |
make check VERBOSE=yes
git diff --exit-code
docker build --platform ${{ matrix.platform }} -t jq-build-${{ env.BIN_NAME }} .
docker cp $(docker create jq-build-${{ env.BIN_NAME }}):/jq ${{ env.BIN_NAME }}
file ${{ env.BIN_NAME }}
Copy link
Member Author

@wader wader Nov 13, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why this got complicated with pcre2 is that we're not vendoring it (i assume we don't want?) so to cross build like before we would have to cross build some version of pcre2 here. the new alternative is that we use docker build with emulation using the our Dockerfile which will end up using pcre2 from debian, that in turn would limit us in archs that can use this approach https://hub.docker.com/_/debian/tags?name=13-slim but maybe it's ok to limit us to whatever debian supports?

- name: Upload Test Logs
if: ${{ failure() }}
uses: actions/upload-artifact@v5
with:
name: test-logs-${{ env.SUFFIX }}
name: test-logs-${{ env.BIN_NAME }}
retention-days: 7
path: |
test-suite.log
tests/*.log
- name: Upload artifacts
uses: actions/upload-artifact@v5
with:
name: jq-${{ env.SUFFIX }}
path: jq-${{ env.SUFFIX }}
name: ${{ env.BIN_NAME }}
path: ${{ env.BIN_NAME }}
if-no-files-found: error
retention-days: 7

Expand All @@ -127,10 +76,10 @@ jobs:
- arm64
include:
- arch: amd64
target: x86_64-apple-darwin
runs_on: macos-15-intel
- arch: arm64
target: arm64-apple-darwin
runs-on: macos-14
runs_on: macos-15
runs-on: ${{ matrix.runs_on }}
env:
LDFLAGS: -dead_strip
SUFFIX: macos-${{ matrix.arch }}
Expand All @@ -143,17 +92,14 @@ jobs:
run: |
# brew update sometimes fails with "Fetching /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask failed!"
brew update || brew update-reset
brew install autoconf automake libtool
- name: Set CC
run: |
echo "CC=clang -target ${{ matrix.target }}$(uname -r)" >> "$GITHUB_ENV"
brew install autoconf automake libtool pcre2
- name: Build
run: |
autoreconf -i
./configure \
--host="${{ matrix.target }}$(uname -r)" \
--disable-docs \
--with-oniguruma=builtin \
--with-oniguruma=no \
--with-pcre2 \
--disable-shared \
--enable-static \
--enable-all-static \
Expand Down Expand Up @@ -228,12 +174,14 @@ jobs:
libtool
pacboy: >-
toolchain:p
pcre2:p
- name: Build
run: |
autoreconf -i
./configure \
--disable-docs \
--with-oniguruma=builtin \
--with-oniguruma=no \
--with-pcre2 \
--disable-shared \
--enable-static \
--enable-all-static \
Expand Down Expand Up @@ -272,7 +220,7 @@ jobs:
- name: Install packages
run: |
sudo apt-get update -qq
sudo apt-get install -y automake autoconf
sudo apt-get install -y automake autoconf libpcre2-dev libpcre2-8-0
- name: Create dist
run: |
autoreconf -i
Expand Down Expand Up @@ -310,11 +258,10 @@ jobs:
merge-multiple: true
- name: Move executables
run: |
mkdir -p linux/{386,amd64,arm64,mips64le,ppc64le,riscv64,s390x}
mkdir -p linux/{386,amd64,arm64,ppc64le,riscv64,s390x}
mv jq-linux-i386 linux/386/jq
mv jq-linux-amd64 linux/amd64/jq
mv jq-linux-arm64 linux/arm64/jq
mv jq-linux-mips64el linux/mips64le/jq
mv jq-linux-ppc64el linux/ppc64le/jq
mv jq-linux-riscv64 linux/riscv64/jq
mv jq-linux-s390x linux/s390x/jq
Expand Down Expand Up @@ -354,7 +301,7 @@ jobs:
context: .
push: ${{ startsWith(github.ref, 'refs/tags/jq-') }}
provenance: false
platforms: linux/386,linux/amd64,linux/arm64,linux/mips64le,linux/ppc64le,linux/riscv64,linux/s390x
platforms: linux/386,linux/amd64,linux/arm64,linux/ppc64le,linux/riscv64,linux/s390x
tags: ${{ steps.metadata.outputs.tags }}
labels: ${{ steps.metadata.outputs.labels }}
- name: Generate signed attestations
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/scanbuild.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@ jobs:
- name: Install packages
run: |
sudo apt-get update -qq
sudo apt-get install -y automake autoconf gdb valgrind clang clang-tools
sudo apt-get install -y automake autoconf gdb valgrind clang clang-tools libpcre2-dev libpcre2-8-0
echo "CC=clang" >> "$GITHUB_ENV"
- name: Build
run: |
autoreconf -i
./configure --enable-valgrind CFLAGS="-g -O0"
./configure --enable-valgrind --with-oniguruma=no --with-pcre2 CFLAGS="-g -O0"
scan-build --keep-going --exclude vendor/ make -j"$(nproc)"
- name: Test
run: |
Expand Down
5 changes: 3 additions & 2 deletions .github/workflows/valgrind.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,15 @@ jobs:
- name: Install packages
run: |
sudo apt-get update
sudo apt-get install -y automake autoconf libtool valgrind
sudo apt-get install -y automake autoconf libtool libpcre2-dev libpcre2-8-0 valgrind
- name: Build
run: |
autoreconf -i
./configure \
--disable-docs \
--enable-valgrind \
--with-oniguruma=builtin
--with-oniguruma=no \
--with-pcre2
make -j"$(nproc)"
file ./jq
- name: Test
Expand Down
7 changes: 5 additions & 2 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM debian:12-slim AS builder
FROM debian:13-slim AS builder

ENV DEBIAN_FRONTEND=noninteractive \
DEBCONF_NONINTERACTIVE_SEEN=true \
Expand All @@ -10,6 +10,8 @@ RUN apt-get update \
build-essential \
autoconf \
libtool \
libpcre2-dev \
libpcre2-8-0 \
git \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
Expand All @@ -19,7 +21,8 @@ COPY . /app
RUN autoreconf -i \
&& ./configure \
--disable-docs \
--with-oniguruma=builtin \
--with-oniguruma=no \
--with-pcre2 \
--enable-static \
--enable-all-static \
--prefix=/usr/local \
Expand Down
10 changes: 8 additions & 2 deletions Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ AM_YFLAGS = --warnings=all -Wno-yacc -d
lib_LTLIBRARIES = libjq.la
libjq_la_SOURCES = ${LIBJQ_SRC}
libjq_la_LIBADD = -lm
libjq_la_LDFLAGS = $(onig_LDFLAGS) -export-symbols-regex '^j[qv]_' -version-info 1:4:0
libjq_la_LDFLAGS = $(onig_LDFLAGS) $(pcre2_LDFLAGS) -export-symbols-regex '^j[qv]_' -version-info 1:4:0

if WIN32
libjq_la_LIBADD += -lshlwapi
Expand Down Expand Up @@ -198,11 +198,17 @@ SUBDIRS = vendor/oniguruma
endif

AM_CFLAGS += $(onig_CFLAGS)
AM_CFLAGS += $(pcre2_CFLAGS)

if WITH_ONIGURUMA
TESTS += tests/onigtest tests/manonigtest
endif

if WITH_PCRE2
TESTS += tests/onigtest tests/manonigtest
TESTS += tests/pcre2test
endif

### Packaging

install-binaries: $(BUILT_SOURCES)
Expand Down Expand Up @@ -230,7 +236,7 @@ EXTRA_DIST = $(DOC_FILES) $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER) \
tests/onig.supp tests/local.supp \
tests/setup tests/torture/input0.json \
tests/optional.test tests/man.test tests/manonig.test \
tests/jq.test tests/onig.test tests/base64.test tests/uri.test \
tests/jq.test tests/onig.test tests/pcre2.test tests/base64.test tests/uri.test \
tests/jq-f-test.sh \
tests/no-main-program.jq tests/yes-main-program.jq

Expand Down
39 changes: 38 additions & 1 deletion configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,44 @@ AC_SUBST(onig_LDFLAGS)
AM_CONDITIONAL([BUILD_ONIGURUMA], [test "x$build_oniguruma" = xyes])
AM_CONDITIONAL([WITH_ONIGURUMA], [test "x$with_oniguruma" != xno])

dnl PCRE2
dnl TODO: proper check
AC_ARG_WITH([pcre2],
[AS_HELP_STRING([--with-pcre2], [Use PCRE2 library])],
[with_pcre2=yes],
[with_pcre2=no])

if test "$with_pcre2" = yes; then
AC_CHECK_PROG(PCRE2_CONFIG, pcre2-config, pcre2-config)
AC_ARG_WITH(pcre2-config,
AS_HELP_STRING([--with-pcre2-config=PATH],
[Location of PCRE2 pcre2-config (auto)]),
[pcre2_config="$withval"],
[pcre2_config=""])

if test "x$pcre2_config" != "x" ; then
AC_MSG_CHECKING(for $pcre2_config)

if test -f $pcre2_config ; then
PCRE2_CONFIG=$pcre2_config
AC_MSG_RESULT(yes)
else
AC_MSG_RESULT(no - searching PATH)
fi
fi
if test "x$PCRE2_CONFIG" = "x"; then
AC_CHECK_PROGS(PCRE2_CONFIG, pcre2-config)
fi
pcre2_CFLAGS=`$PCRE2_CONFIG --cflags`
pcre2_LDFLAGS=`$PCRE2_CONFIG --libs8`
AC_SUBST(pcre2_CFLAGS)
AC_SUBST(pcre2_LDFLAGS)
dnl condtional?
AC_DEFINE([HAVE_PCRE2],1,[Define to 1 if the system includes pcre2])
fi

AM_CONDITIONAL([WITH_PCRE2], [test "x$with_pcre2" != xno])

AC_CONFIG_MACRO_DIRS([config/m4 m4])
AC_CONFIG_FILES([Makefile libjq.pc])
AC_OUTPUT

Loading
Loading