added several improvements (#107)

* static compiler start

* successfully called shared library

* several changes and improvements

* python file is obsolete

* small cleanups

* improve Artifacts handling

* missed libdl test dependency

test run through now again

* added julia style guide as formatter option

* adding pre-commit hooks for auto formatting

* simplified local setup using just

* need to include source command so that tools are also available on first run

* libdl 1.10

Author: schlichtanders

.JuliaFormatter.toml

@@ -0,0 +1,2 @@
+# see https://domluna.github.io/JuliaFormatter.jl/stable/ for a list of options
+style = "blue"

.dockerignore

@@ -0,0 +1,17 @@
+packagecompiler/app
+packagecompiler/app.*
+.github
+.vscode
+.CondaPkg
+docs
+test
+tmp
+**/tmp
+AUTHORS
+codemeta.json
+Dockerfile
+Dockerfile.*
+LICENSE
+README.md
+staticcompiler/lib
+staticcompiler/lib.*

.gitignore

@@ -3,7 +3,9 @@
 *.jl.mem
 /docs/Manifest.toml
 /docs/build/
-Manifest.toml
-tmp/
+/Manifest.toml
+tmp
 /packagecompiler/app
-/packagecompiler/app.backup
+*.backup
+.CondaPkg
+.vscode

.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v2.3.0
+  hooks:
+  - id: check-yaml
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+
+- repo: https://github.com/qiaojunfeng/pre-commit-julia-format
+  rev: v0.2.0
+  hooks:
+  - id: julia-format

test/Artifacts.toml Artifacts.tomltest/Artifacts.toml renamed to Artifacts.toml

@@ -1,5 +1,11 @@
+# all entries marked with `lazy = true` won't be downloaded on package installation
+# the rest will be downloaded
+# hence we can simply mark all test dependencies as lazy = true and have a single Artifacts.toml for everything
+
 [rqatestdata]
 git-tree-sha1 = "1dc9d7aa975de4f5ecc652f34d6bbee706fcb7bd"
+# mark all test Artifacts as lazy
+lazy = true
 
     [[rqatestdata.download]]
     url = "https://github.com/EarthyScience/RQADeforestationTestData/archive/refs/tags/v2.0.tar.gz"

Dockerfile

@@ -1,6 +1,26 @@
+FROM julia:1.11-bookworm AS packagecompiled
+
+RUN --mount=type=cache,target=/var/cache/apt \
+    --mount=type=cache,target=/var/lib/apt \ 
+    apt -y update && apt -y install build-essential curl ca-certificates
+
+# from Debian version 13 onwards, "just" can simply be installed via apt install
+RUN curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /usr/local/bin
+
+WORKDIR /repo
+COPY . .
+RUN --mount=type=cache,target=/root/.julia \
+    julia --project=. -e "import Pkg; Pkg.instantiate()"
+RUN --mount=type=cache,target=/root/.julia \
+    julia --project="packagecompiler" -e "import Pkg; Pkg.instantiate()"
+RUN --mount=type=cache,target=/root/.julia \
+    just packagecompile
+
+# having precompiled everything we can simply copy the binaries to its slim docker image
 FROM debian:bookworm-slim
 WORKDIR /work
-COPY packagecompiler/app/bin /usr/bin
-COPY packagecompiler/app/lib /usr/lib
-COPY packagecompiler/app/share /usr/share
-ENTRYPOINT ["RQADeforestation"]
+COPY --from=packagecompiled /repo/packagecompiler/app/bin /usr/bin
+COPY --from=packagecompiled /repo/packagecompiler/app/lib /usr/lib
+COPY --from=packagecompiled /repo/packagecompiler/app/share /usr/share
+ENTRYPOINT ["RQADeforestation"]
+

Dockerfile.staticcompiler

@@ -0,0 +1,20 @@
+FROM julia:1.10
+
+# installing C dependencies to build julia using StaticCompiler.jl
+# adapted from stackoverflow https://askubuntu.com/a/1512328
+RUN --mount=type=cache,target=/var/cache/apt \
+    --mount=type=cache,target=/var/lib/apt \ 
+    apt -y update && apt -y install build-essential gcc libc-dev
+
+# Set work directory
+WORKDIR /app
+# Copy project files
+COPY . .
+# work inside staticcompiler subfolder
+WORKDIR /app/staticcompiler 
+# Precompile packages
+
+RUN --mount=type=cache,target=/root/.julia \
+    julia --project=. -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()'
+RUN --mount=type=cache,target=/root/.julia \
+    julia --project=. staticcompile_rqatrend.jl

Project.toml

@@ -40,32 +40,45 @@ YAXArrays = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c"
 Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99"
 
 [compat]
+AllocCheck = "0.2.2"
+Aqua = "0.8.13"
 ArchGDAL = "0.10"
 ArgParse = "1"
+BenchmarkTools = "1.6.0"
 ConstructionBase = "1"
+Dates = "1.10"
 DimensionalData = "0.29"
 DiskArrayEngine = "0.1, 0.2"
 DiskArrayTools = "0.1"
 DiskArrays = "0.4"
 Distances = "0.10"
+Distributed = "1.10"
 Distributions = "0.25"
+Documenter = "1.12.0"
 Extents = "0.1"
 FillArrays = "1"
 GDAL = "1"
 GeoFormatTypes = "0.4"
 Glob = "1"
 KML = "0.2"
+Libdl = "1.10"
 LinearAlgebra = "1.10"
 LoggingExtras = "1"
 Missings = "1"
 NetCDF = "0.12"
+Pkg = "1.10"
 Proj = "1"
+PythonCall = "0.9.25"
+Random = "1.10"
 Rasters = "0.12,0.13, 0.14"
 RecurrenceAnalysis = "2"
 StaticArrays = "1"
+StaticTools = "0.8.11"
 Statistics = "1.10"
 StatsBase = "0.34"
-TestItems = "1.0.0"
+Test = "1.10"
+TestItemRunner = "1.0"
+TestItems = "1.0"
 TimeseriesSurrogates = "2"
 UnicodePlots = "3"
 YAXArrayBase = "0.6, 0.7"
@@ -75,11 +88,16 @@ julia = "1.11"
 
 [extras]
 AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a"
+Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+StaticTools = "86c06d3c-3f03-46de-9781-57580aa96d0a"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
+Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
 [targets]
-test = ["Test", "TestItemRunner", "Pkg", "Random", "AllocCheck", "BenchmarkTools"]
+test = ["Test", "TestItemRunner", "Pkg", "Random", "AllocCheck", "BenchmarkTools", "Aqua", "Documenter", "StaticTools", "PythonCall", "Libdl"]

README_DEVELOPER.md

@@ -0,0 +1,13 @@
+# Welcome to RQADeforestation.jl
+
+This README collects important information for local development of the package.
+
+# Local setup
+
+Install `just` into your system, following its README [installation instructions](https://github.com/casey/just?tab=readme-ov-file#installation).
+
+Then run
+```bash
+just init
+```
+This will for instance setup pre-commit to enforce the standard Julia styling [blue](https://github.com/JuliaDiff/BlueStyle).

justfile

@@ -1,6 +1,24 @@
+# just configurations
+# setting tempdir to local path makes julia macro artifact"..." work
+set tempdir := "./tmp"
+
+# global variables
+docker_image_name := "rqatest"
+
 default:
   just --list
 
+init:
+    #!/usr/bin/env bash
+    set -euxo pipefail
+    # setup pre-commit hooks via pythons uv tooling manager
+    curl -LsSf https://astral.sh/uv/install.sh | sh
+    source ~/.local/bin/env
+    uv tool install pre-commit
+    uvx pre-commit install
+    # the julia formatting hook needs JuliaFormatter installed globally
+    julia -e 'import Pkg; Pkg.add("JuliaFormatter")'
+
 # create precompile statements used for `just packagecompile`
 precompilestatements:
     julia --project -e 'import Pkg; Pkg.test(julia_args=`--trace-compile=packagecompiler/precompile_statements.jl`)'
@@ -9,11 +27,11 @@ precompilestatements:
 packagecompile:
     #!/usr/bin/env -S julia --project=packagecompiler
     using PackageCompiler
-    if isdir("packagecompiler/app") 
+    if isdir("packagecompiler/app")
         rm("packagecompiler/app.backup", recursive=true, force=true)
         mv("packagecompiler/app", "packagecompiler/app.backup")
     end
-    PackageCompiler.create_app(".", "packagecompiler/app"; 
+    PackageCompiler.create_app(".", "packagecompiler/app";
         precompile_statements_file="packagecompiler/precompile_statements.jl",
         # see https://github.com/JuliaLang/PackageCompiler.jl/issues/994
         include_lazy_artifacts=true,
@@ -23,19 +41,50 @@ packagecompile:
     # lets have an easy check whether this actually worked
     touch("packagecompiler/app/done")
 
-# test the app `packagecompiler/app/RQADeforestation` with testdata, writing data to `test/tmp/apptestdata`
-testapp:
+# downloads the Artifact test data to tmp/apptestdata
+download-test-data $tmpdir="tmp/apptestdata":
+    #!/usr/bin/env -S julia --project
+    import Pkg: Artifacts.@artifact_str
+    testdatapath = artifact"rqatestdata/RQADeforestationTestData-2.0"
+    testdir = ENV["tmpdir"]
+    rm(testdir, recursive=true, force=true)
+    mkpath(testdir)
+    cp(testdatapath, joinpath(testdir, "in"))
+
+# test the app `packagecompiler/app/RQADeforestation` with testdata, writing data to `tmp/apptestdata`
+test-app $tmpdir="tmp/apptestdata": (download-test-data tmpdir)
+    #!/usr/bin/env bash
+    set -euxo pipefail
+    indir="$tmpdir/in"
+    outdir="$tmpdir/out.zarr"
+    ./packagecompiler/app/bin/RQADeforestation --tile E051N018T3 --continent EU --start-date "2021-01-01" --end-date "2022-01-01" --in-dir "$indir" --out-dir "$outdir"
+
+# builds the standard docker
+build-docker:
+    docker build -t "{{docker_image_name}}" -f Dockerfile .
+
+# tests the build docker image using the Artifact test data, writing data to `tmp/apptestdata`
+test-docker $tmpdir="tmp/apptestdata": (download-test-data tmpdir)
+    #!/usr/bin/env bash
+    set -euxo pipefail
+    indir="$tmpdir/in"
+    outdir="$tmpdir/out.zarr"
+    docker run --user $(id -u):$(id -g) --rm -v "$PWD/$tmpdir":"/$tmpdir" "{{docker_image_name}}" --tile E051N018T3 --continent EU --start-date "2021-01-01" --end-date "2022-01-01" --in-dir "/$indir" --out-dir "/$outdir"
+
+# compiles rqatrend to its own c-library using StaticCompiler.jl
+staticcompile:
     #!/usr/bin/env bash
-    cd test
-    indir="tmp/apptestdata/in"
-    outdir="tmp/apptestdata/out.zarr"
-    julia --project -e '
-        import Pkg: Artifacts.@artifact_str, ensure_artifact_installed
-        ensure_artifact_installed("rqatestdata", "Artifacts.toml")
-        testdatapath = joinpath(artifact"rqatestdata", "RQADeforestationTestData-1.0")
-        testdir = dirname(ARGS[1])
-        rm(testdir, recursive=true, force=true)
-        mkpath(testdir)
-        cp(testdatapath, ARGS[1])
-    ' -- "$indir"
-    ../packagecompiler/app/bin/RQADeforestation --tile E051N018T3 --continent EU --in-dir "$indir" --out-dir "$outdir"
+    set -euxo pipefail
+    if [ -d staticcompiler/lib ]; then
+        if [ -d staticcompiler/lib.backup ]; then
+            rm -rf staticcompiler/lib.backup
+        fi
+        mv staticcompiler/lib staticcompiler/lib.backup
+    fi
+    # using progress plain is important as staticcompiler.jl
+    # outputs warnings instead of errors if things may not work
+    docker build --progress=plain -t temp-image -f Dockerfile.staticcompiler .
+    docker create --name temp-container temp-image
+    docker cp temp-container:/app/staticcompiler/lib "$PWD/staticcompiler/lib"
+    docker rm temp-container
+    docker rmi temp-image