diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 3df0cf4..1678b31 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -87,8 +87,11 @@ jobs: .\gradlew.bat sample:clean sample:shadowJar java -jar sample/build/libs/sample-all.jar - docker-checks: + docker-checks-matrix: runs-on: ubuntu-24.04 + strategy: + matrix: + variant: ["debian-12", "ubuntu-2404", "ubuntu-2404-jemalloc"] steps: - name: Checkout uses: actions/checkout@v4 @@ -105,4 +108,11 @@ jobs: gradle-version: 8.14 - name: Run checks - run: ./run_docker_tests.sh \ No newline at end of file + run: ./run_single_docker_tests.sh ${{ matrix.variant }} + + docker-checks: + runs-on: ubuntu-24.04 + name: Verify all Docker checks passed + needs: [docker-checks-matrix] + steps: + - run: exit 0 \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c0db903..3759a5d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,6 +15,10 @@ library is already used by some larger businesses (which is cool!), and I care a Please discuss any bigger changes with me **before** submitting a Pull Request - I can help you refine your idea better that way, and I don't want to waste anybody's time: [Discussions](https://github.com/lopcode/vips-ffm/discussions). +As part of a pull request I will probably edit commits on the branch, and will squash them down, but will be careful to +retain your contributor metadata so you're named appropriately as a contributor on GitHub. GitHub Actions workflows to +run the project's tests require approval - I'll do this when I'm reviewing the PR. + I haven't currently defined a code of conduct for this project specifically, but please refer to the CoC [in libvips](https://github.com/libvips/libvips/blob/master/CODE_OF_CONDUCT.md) for guidance on expected behaviour. diff --git a/README.md b/README.md index af64b7f..8f6be1d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ Fast, safe, complete [libvips](https://github.com/libvips/libvips) bindings for Supports a vast range of image formats, including HEIC, JXL, WebP, PNG, JPEG, and more. Pronounced "vips (like zips) eff-eff-emm". The project is relatively new, but aims to be production ready. Tested on macOS 14, Windows 11, and Linux -(Ubuntu 24.04, Debian 12.1). Should work on any architecture you can use libvips and Java on (arm64/amd64/etc). +(Ubuntu 24.04, Debian 12.1, with and without jemalloc). Should work on any architecture you can use libvips and +Java on (arm64/amd64/etc). Uses the "Foreign Function & Memory API" ([JEP 454](https://openjdk.org/jeps/454)), and the "Class-File API" ([JEP 457](https://openjdk.org/jeps/457)) released in JDK 22. Built in such a way that it's usually the fastest image processing library available for Java. @@ -57,7 +58,7 @@ import app.photofox.vipsffm.enums.VipsAccess // Call once to initialise libvips when your program starts, from any thread // Note that by default this blocks untrusted operations (like loading PDFs) -// Use `Vips.init(true, ...)` to permit untrusted operations +// See the "Allowing untrusted operations" section below to read about permitting untrusted operations Vips.init() // Use `Vips.run` to wrap your usage of the API, and get an arena with an appropriate lifetime to use @@ -193,6 +194,60 @@ like Android where it's hard to set the system library path), you can do so usin * glib: `vipsffm.libpath.glib.override` * gobject: `vipsffm.libpath.gobject.override` +## Operationalisation + +libvips maintain [a checklist](https://www.libvips.org/API/8.17/developer-checklist.html) of things to be aware of when +using the library. Of particular note for vips-ffm is memory usage - especially if the library is used for long-running +application (like a server). + +### Operation cache + +At the time of writing, libvips maintains a cache of the 100 most recent operations ([see docs](https://www.libvips.org/API/8.17/how-it-works.html#operation-cache)). +If running an image proxy, or something that processes lots of different images, you won't see any benefit, and can +disable it: + +```java +Vips.init(); +Vips.disableOperationCache(); +``` + +### Memory allocation + +On glibc-based Linux systems (e.g. Debian, Red Hat), the default memory allocator performs poorly for long-running, +multithreaded processes with frequent small allocations. Using an alternative allocator like jemalloc can reduce the +off-heap footprint of the JVM when using libvips. + +Note that the jemalloc project is going through [some turbulence](https://jasone.github.io/2025/06/12/jemalloc-postmortem/) +at the moment. Facebook have [forked it](https://github.com/facebook/jemalloc), though its maintenance status is +currently unknown. + +An example of using jemalloc on Ubuntu: +1. Install jemalloc + ```sh + apt install libjemalloc2 + ``` +2. Set the `LD_PRELOAD` environment variable before running your application. + ```sh + ln -sT "$(readlink -e /usr/lib/*/libjemalloc.so.2)" /usr/local/lib/libjemalloc.so # symlink jemalloc to a standard location + export LD_PRELOAD=/usr/local/lib/libjemalloc.so + java -jar ... + ``` + +### Allowing untrusted operations + +By default, vips-ffm sets the "block untrusted operations" flag in libvips, in an attempt to be "secure by default". +This includes blocking things like the imagemagick and PDF loaders. If you get an error relating to "operation is +blocked", then the operation you're trying to use is marked as untrusted in libvips. + +If you need to work with operations and formats that are marked as "untrusted" in libvips, you can permit them +explicitly: +```java +Vips.allowUntrustedOperations(); +``` + +See the [libvips docs](https://www.libvips.org/API/8.17/func.block_untrusted_set.html) for guidance on figuring out what +loaders and operations are marked as trusted or untrusted. + ## Project goals Ideas and suggestions are welcome, but please make sure they fit in to these goals, or you have a good argument about @@ -224,4 +279,4 @@ Thank you for being enthusiastic about the project! * And only after a GitHub Release is made * Run `./publish_release_to_maven_central.sh ` -[1]: https://docs.oracle.com/en/java/javase/23/core/memory-segments-and-arenas.html \ No newline at end of file +[1]: https://docs.oracle.com/en/java/javase/23/core/memory-segments-and-arenas.html diff --git a/core/src/main/java/app/photofox/vipsffm/Vips.java b/core/src/main/java/app/photofox/vipsffm/Vips.java index 130db8f..f9fe79e 100644 --- a/core/src/main/java/app/photofox/vipsffm/Vips.java +++ b/core/src/main/java/app/photofox/vipsffm/Vips.java @@ -17,6 +17,9 @@ public static void init(boolean allowUntrusted, boolean detectLeaks) { VipsHelper.leak_set(detectLeaks); } + /// Provides a scoped arena to provide a memory boundary for running libvips operations + /// + /// After the scope has ended, any memory allocated whilst using libvips within it will be freed public static void run(VipsRunnable runnable) { try (var arena = Arena.ofConfined()) { runnable.run(arena); @@ -26,4 +29,22 @@ public static void run(VipsRunnable runnable) { public static void shutdown() { VipsHelper.shutdown(); } + + /// Permits untrusted operations, such as loading PDFs + /// + /// vips-ffm blocks these by default - see the [libvips docs](https://www.libvips.org/API/8.17/func.block_untrusted_set.html) + /// for guidance + public static void allowUntrustedOperations() { + VipsHelper.block_untrusted_set(false); + } + + /// Disables the libvips operations cache + /// + /// At the time of writing libvips caches 100 operations by default, which might not be useful in long-running + /// applications (like servers). + /// + /// See also: [libvips docs](https://www.libvips.org/API/8.17/how-it-works.html#operation-cache) + public static void disableOperationCache() { + VipsHelper.cache_set_max(0); + } } diff --git a/docker_tests/debian-12/Dockerfile b/docker_tests/debian-12/Dockerfile index 2ca8a0c..3160911 100644 --- a/docker_tests/debian-12/Dockerfile +++ b/docker_tests/debian-12/Dockerfile @@ -6,7 +6,7 @@ ENV PATH="${JAVA_HOME}/bin:${PATH}" COPY sample /opt/sample COPY run_samples.sh /opt/run_samples.sh -RUN apt update && apt install libvips-dev -y +RUN apt update && apt install --no-install-recommends --yes libvips-dev libvips-tools libjemalloc2 RUN vips --version WORKDIR /opt diff --git a/docker_tests/ubuntu-2404-jemalloc/Dockerfile b/docker_tests/ubuntu-2404-jemalloc/Dockerfile new file mode 100644 index 0000000..402b893 --- /dev/null +++ b/docker_tests/ubuntu-2404-jemalloc/Dockerfile @@ -0,0 +1,15 @@ +FROM ubuntu:24.04 +ENV JAVA_HOME=/opt/java/openjdk +COPY --from=eclipse-temurin:23 $JAVA_HOME $JAVA_HOME +ENV PATH="${JAVA_HOME}/bin:${PATH}" + +COPY sample /opt/sample +COPY run_samples.sh /opt/run_samples.sh + +RUN apt update && apt install --no-install-recommends --yes libvips-dev libvips-tools libjemalloc2 +RUN ln -sT "$(readlink -e /usr/lib/*/libjemalloc.so.2)" /usr/local/lib/libjemalloc.so + +RUN vips --version + +WORKDIR /opt +CMD ./run_samples.sh \ No newline at end of file diff --git a/docker_tests/ubuntu-2404/Dockerfile b/docker_tests/ubuntu-2404/Dockerfile index 7ebc91b..754f92d 100644 --- a/docker_tests/ubuntu-2404/Dockerfile +++ b/docker_tests/ubuntu-2404/Dockerfile @@ -6,7 +6,7 @@ ENV PATH="${JAVA_HOME}/bin:${PATH}" COPY sample /opt/sample COPY run_samples.sh /opt/run_samples.sh -RUN apt update && apt install libvips-dev -y +RUN apt update && apt install --no-install-recommends --yes libvips-dev libvips-tools libjemalloc2 RUN vips --version WORKDIR /opt diff --git a/run_docker_tests.sh b/run_docker_tests.sh index a7ec7b5..b29ca3b 100755 --- a/run_docker_tests.sh +++ b/run_docker_tests.sh @@ -2,18 +2,11 @@ set -eou pipefail echo "building samples..." -./gradlew sample:clean sample:shadowJar +./gradlew sample:shadowJar -echo "running docker tests..." -WORKSPACE_DIR="$PWD" +echo "running all docker tests..." -docker_tests=("debian-12" "ubuntu-2404") +docker_tests=("ubuntu-2404-jemalloc" "debian-12" "ubuntu-2404") for docker_test in "${docker_tests[@]}"; do - echo "testing \"$docker_test\"" - pushd "docker_tests/$docker_test" - cp -r "$WORKSPACE_DIR"/sample . - cp "$WORKSPACE_DIR"/run_samples.sh . - docker build -t "vips-ffm-$docker_test-test" . - docker run "vips-ffm-$docker_test-test" - popd + ./run_single_docker_tests.sh "$docker_test" || (echo "test failed" && exit 1) done \ No newline at end of file diff --git a/run_samples.sh b/run_samples.sh index 5df9e0e..285273b 100755 --- a/run_samples.sh +++ b/run_samples.sh @@ -15,6 +15,11 @@ if [[ "$OSTYPE" == "darwin"* ]]; then export JAVA_PATH_OPTS="-Dvipsffm.libpath.vips.override=/opt/homebrew/lib/libvips.dylib" fi +if test -f /usr/local/lib/libjemalloc.so; then + echo "found jemalloc - using it" + export LD_PRELOAD=/usr/local/lib/libjemalloc.so +fi + echo "running samples..." java $JAVA_PATH_OPTS -jar sample/build/libs/sample-all.jar 2>&1 | tee sample_output.log diff --git a/run_single_docker_tests.sh b/run_single_docker_tests.sh new file mode 100755 index 0000000..7ac0187 --- /dev/null +++ b/run_single_docker_tests.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -eou pipefail + +if [ -z "$1" ]; then + echo "usage: ./run_single_docker_tests.sh ubuntu-2404" + exit 1 +fi + +echo "building samples..." +./gradlew sample:shadowJar + +echo "verifying docker test variant \"$1\" exists..." +if [ ! -d "docker_tests/$1" ]; then + echo "could not find docker test variant - exiting" + exit 1 +fi + +echo "running docker tests for variant $1..." +WORKSPACE_DIR="$PWD" + +pushd "docker_tests/$1" +cp -r "$WORKSPACE_DIR"/sample . +cp "$WORKSPACE_DIR"/run_samples.sh . +docker build --progress=plain -t "vips-ffm-$1-test" . +docker run "vips-ffm-$1-test" +popd \ No newline at end of file