feat: impl pre-processing encodings #8740

Workflow file for this run

	name: CI

	on:
	push:
	branches: [ main ]
	pull_request:
	branches: [ main ]
	release:
	types: [ published ]
	workflow_dispatch:

	defaults:
	run:
	shell: bash

	permissions:
	contents: read

	jobs:
	lint-unit-test-js:
	name: Lint and Unit test the frontend
	permissions:
	contents: read
	runs-on: ubuntu-latest
	needs:
	- lint-rust # not because it is needed, but because we want less concurrent jobs competing
	steps:
	- name: Checkout sources
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with: { persist-credentials: false }
	- uses: taiki-e/install-action@0abfcd587b70a713fdaa7fb502c885e2112acb15 # v2.75.7
	with: { tool: 'just' }
	- run: npm clean-install --no-fund
	working-directory: martin/martin-ui
	- run: just ui::biome
	- run: just ui::test
	- run: just ui::type-check
	unit-test-rust:
	name: Unit test the server (${{ matrix.os }})
	permissions:
	contents: read
	runs-on: ${{ matrix.os }}
	needs:
	- lint-rust # not because it is needed, but because we want less concurrent jobs competing
	strategy:
	fail-fast: false
	matrix:
	# 1. Linux-only for PRs (faster feedback)
	# 2. Run on all three OS platforms when merging to main
	os: ${{ fromJSON('["ubuntu-latest", "macos-latest"]') \|\| (github.ref == 'refs/heads/main' && fromJSON('["ubuntu-latest", "macos-latest", "windows-latest"]')) }}
	steps:
	- run: rustup update stable && rustup default stable
	- uses: BRAINSia/free-disk-space@7048ffbf50819342ac964ef3998a51c2564a8a75 # v2.1.3
	if: runner.os == 'Linux'
	with:
	tool-cache: false
	mandb: true
	android: true
	dotnet: true
	haskell: true
	large-packages: true
	docker-images: true
	swap-storage: true
	- name: Checkout sources
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with: { persist-credentials: false }
	- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
	if: github.event_name != 'release' && github.event_name != 'workflow_dispatch'
	with:
	key: ${{ matrix.os }}
	- uses: taiki-e/install-action@0abfcd587b70a713fdaa7fb502c885e2112acb15 # v2.75.7
	with: { tool: 'just' }
	- name: Run cargo test
	run: \|
	set -x
	cargo test --package martin-tile-utils
	cargo test --package mbtiles --no-default-features
	cargo test --package mbtiles
	cargo test --package martin
	cargo test --package martin-core
	env:
	AWS_SKIP_CREDENTIALS: 1
	AWS_REGION: eu-central-1
	- run: cargo test --doc
	lint-rust:
	name: Lint the Rust code
	permissions:
	contents: read
	runs-on: ubuntu-latest
	steps:
	- run: rustup update stable && rustup default stable
	- run: rustup component add clippy rustfmt
	- name: Checkout sources
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with: { persist-credentials: false }
	- uses: taiki-e/install-action@0abfcd587b70a713fdaa7fb502c885e2112acb15 # v2.75.7
	with: { tool: 'just' }
	- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
	if: github.event_name != 'release' && github.event_name != 'workflow_dispatch' && github.ref != 'refs/heads/main'
	- run: just env-info
	- run: just clippy
	- run: just check
	- run: just check-doc
	lint-rust-dependencies:
	name: Lint Rust dependencies
	permissions:
	contents: read
	needs:
	- lint-rust # not because it is needed, but because we want less concurrent jobs competing
	runs-on: ubuntu-latest
	steps:
	- run: rustup default nightly
	- name: Checkout sources
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with: { persist-credentials: false }
	- uses: taiki-e/install-action@0abfcd587b70a713fdaa7fb502c885e2112acb15 # v2.75.7
	with: { tool: 'cargo-shear,just' }

	- run: just shear
	build-aarch64-unknown-linux-musl:
	uses: ./.github/workflows/build-binary.yml
	with:
	target: aarch64-unknown-linux-musl
	os: ubuntu-latest
	musl: true
	release_mode: ${{ github.event_name != 'pull_request' }}

	build-x86_64-unknown-linux-musl:
	uses: ./.github/workflows/build-binary.yml
	with:
	target: x86_64-unknown-linux-musl
	os: ubuntu-latest
	musl: true
	release_mode: ${{ github.event_name != 'pull_request' }}

	docker-build-test:
	name: Build and test docker images
	runs-on: ubuntu-latest
	needs: [ build-aarch64-unknown-linux-musl, build-x86_64-unknown-linux-musl ]
	permissions:
	id-token: write
	attestations: write
	packages: write
	contents: read
	env:
	# PG_* variables are used by psql
	PGDATABASE: test
	PGHOST: localhost
	PGUSER: postgres
	PGPASSWORD: postgres
	services:
	postgres:
	image: postgis/postgis:18-3.6
	ports:
	- 5432/tcp
	options: >-
	-e POSTGRES_DB=test
	-e POSTGRES_USER=postgres
	-e POSTGRES_PASSWORD=postgres
	-e PGDATABASE=test
	-e PGUSER=postgres
	-e PGPASSWORD=postgres
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	--entrypoint sh
	postgis/postgis:18-3.6
	-c "exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key"

	steps:
	- name: Checkout sources
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with: { persist-credentials: false }
	- name: Init database
	run: tests/fixtures/initdb.sh
	env:
	PGPORT: ${{ job.services.postgres.ports[5432] }}
	- uses: taiki-e/install-action@0abfcd587b70a713fdaa7fb502c885e2112acb15 # v2.75.7
	with: { tool: sqlx-cli }

	# we want to test our docker images before we push them to the registry
	# this is done by
	# - building multi-arch images
	# - executing the test script against each arch via docker
	# - pushing them to the registry
	#
	# We need the containerd image store, as otherwise multi-arch cannot be loaded => https://github.com/docker/buildx/blob/master/docs/reference/buildx_build.md#docker
	# > The default image store in Docker Engine doesn't support loading multi-platform images.
	# > You can enable the containerd image store, or push multi-platform images is to directly push to a registry
	- name: Set up Docker
	uses: docker/setup-docker-action@1a6edb0ba9ac496f6850236981f15d8f9a82254d # v5.0.0
	with:
	daemon-config: \|
	{
	"features": {
	"containerd-snapshotter": true
	}
	}
	- name: Set up QEMU
	uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
	# https://github.com/docker/setup-qemu-action
	with: { platforms: 'linux/amd64,linux/arm64' }
	- name: Set up gdal-bin and sqlite3-tools
	run: sudo apt-get update && sudo apt-get install -y gdal-bin sqlite3-tools
	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
	# https://github.com/docker/setup-buildx-action
	with: { install: true, platforms: 'linux/amd64,linux/arm64' }
	- name: Download build artifact build-aarch64-unknown-linux-musl
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with: { name: 'build-aarch64-unknown-linux-musl', path: 'target_releases/linux/arm64/' }
	- name: Mark target_releases/linux/arm64/* as executable
	run: chmod +x target_releases/linux/arm64/*
	- name: Download build artifact build-x86_64-unknown-linux-musl
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with: { name: 'build-x86_64-unknown-linux-musl', path: 'target_releases/linux/amd64/' }
	- name: Mark target_releases/linux/amd64/* as executable
	run: chmod +x target_releases/linux/amd64/*
	- name: Start NGINX
	uses: nyurik/action-setup-nginx@612ee9cb839ad727832cda16359452e7e14dd521 # v1.1
	id: nginx
	with: { port: '5412', output-unix-paths: 'yes' }
	- name: Copy static files
	env:
	HTML_DIR: ${{ steps.nginx.outputs.html-dir }}
	run: cp -r tests/fixtures/pmtiles2/* "$HTML_DIR"
	- name: Prepare tag
	id: prepare_tag
	env:
	EVENT_NAME: ${{ github.event_name }}
	RELEASE_TAG: ${{ github.event.release.tag_name }}
	run: \|
	if [ "$EVENT_NAME" = "release" ]; then
	TAG="$RELEASE_TAG"
	else
	TAG="nightly"
	fi

	# Strip prefix "martin-v" if present
	echo "short_tag=${TAG#martin-v}" >> "$GITHUB_OUTPUT"
	- name: Docker meta
	id: docker_meta
	uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
	# https://github.com/docker/metadata-action
	with:
	images: ghcr.io/${{ github.repository }}
	tags: \|
	type=raw,value=${{ steps.prepare_tag.outputs.short_tag }}
	type=raw,value=latest,enable=${{ github.event_name == 'release' && startsWith(github.event.release.tag_name, 'martin-v') }}
	labels: \|
	org.opencontainers.image.created={{commit_date 'YYYY-MM-DDTHH:mm:ss.SSS[Z]'}}
	org.opencontainers.image.authors=Yuri Astrakhan, Stepan Kuzmin and MapLibre contributors
	org.opencontainers.image.url=https://github.com/${{ github.repository }}
	org.opencontainers.image.documentation=https://maplibre.org/martin/
	org.opencontainers.image.source=${{ github.event.repository.html_url }}
	org.opencontainers.image.revision=${{ github.sha }}
	org.opencontainers.image.version={{version}}
	org.opencontainers.image.vendor=maplibre
	org.opencontainers.image.licenses=Apache-2.0 OR MIT
	org.opencontainers.image.description=Blazing fast and lightweight tile server with PostGIS, MBTiles, and PMTiles support
	org.opencontainers.image.title=${{ github.event.repository.name }}
	org.opencontainers.image.source=https://github.com/maplibre/martin
	annotations: \|
	org.opencontainers.image.created={{commit_date 'YYYY-MM-DDTHH:mm:ss.SSS[Z]'}}
	org.opencontainers.image.authors=Yuri Astrakhan, Stepan Kuzmin and MapLibre contributors
	org.opencontainers.image.url=https://github.com/${{ github.repository }}
	org.opencontainers.image.documentation=https://maplibre.org/martin/
	org.opencontainers.image.source=${{ github.event.repository.html_url }}
	org.opencontainers.image.revision=${{ github.sha }}
	org.opencontainers.image.version={{version}}
	org.opencontainers.image.vendor=maplibre
	org.opencontainers.image.licenses=Apache-2.0 OR MIT
	org.opencontainers.image.description=Blazing fast and lightweight tile server with PostGIS, MBTiles, and PMTiles support
	org.opencontainers.image.title=${{ github.event.repository.name }}
	org.opencontainers.image.source=https://github.com/maplibre/martin
	env:
	DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
	- name: Build linux/arm64,linux/amd64 Docker image
	uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
	# https://github.com/docker/build-push-action
	with:
	provenance: mode=max
	sbom: true
	context: .
	file: .github/files/multi-platform.Dockerfile
	push: false
	load: true
	tags: ${{ github.repository }}:linux
	annotations: ${{ steps.docker_meta.outputs.annotations }}
	labels: ${{ steps.docker_meta.outputs.labels }}
	platforms: linux/arm64,linux/amd64

	- name: Test linux/arm64 Docker image
	run: \|
	PLATFORM=linux/arm64
	TAG=${{ github.repository }}:linux
	export MARTIN_BUILD_ALL=-
	export MARTIN_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -e AWS_REGION=eu-central-1 -e AWS_SKIP_CREDENTIALS=1 -e RUST_LOG_FORMAT=bare -v $PWD/tests:/tests $TAG"
	export MARTIN_CP_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -e AWS_REGION=eu-central-1 -e AWS_SKIP_CREDENTIALS=1 -e RUST_LOG_FORMAT=bare -v $PWD/tests:/tests --entrypoint /usr/local/bin/martin-cp $TAG"
	export MBTILES_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -e AWS_REGION=eu-central-1 -e AWS_SKIP_CREDENTIALS=1 -v $PWD/tests:/tests --entrypoint /usr/local/bin/mbtiles $TAG"
	tests/test.sh
	env:
	DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require
	- name: Test linux/amd64 Docker image
	run: \|
	PLATFORM=linux/amd64
	TAG=${{ github.repository }}:linux
	export MARTIN_BUILD_ALL=-
	export MARTIN_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -e AWS_REGION=eu-central-1 -e AWS_SKIP_CREDENTIALS=1 -e RUST_LOG_FORMAT=bare -v $PWD/tests:/tests $TAG"
	export MARTIN_CP_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -e AWS_REGION=eu-central-1 -e AWS_SKIP_CREDENTIALS=1 -e RUST_LOG_FORMAT=bare -v $PWD/tests:/tests --entrypoint /usr/local/bin/martin-cp $TAG"
	export MBTILES_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -e AWS_REGION=eu-central-1 -e AWS_SKIP_CREDENTIALS=1 -v $PWD/tests:/tests --entrypoint /usr/local/bin/mbtiles $TAG"
	tests/test.sh
	env:
	DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require

	- name: Save test output (on error)
	if: failure()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: failed-test-output-docker
	path: \|
	tests/output/*
	target/test_logs/*
	retention-days: 5

	- name: Login to GitHub Docker registry
	if: github.event_name == 'push' \|\| (github.event_name == 'release' && startsWith(github.event.release.tag_name, 'martin-v'))
	uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
	# https://github.com/docker/login-action
	with:
	registry: ghcr.io
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Push the Docker image
	id: docker_push
	if: github.event_name == 'push' \|\| (github.event_name == 'release' && startsWith(github.event.release.tag_name, 'martin-v'))
	uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
	with:
	provenance: mode=max
	sbom: true
	context: .
	file: .github/files/multi-platform.Dockerfile
	push: true
	load: false
	tags: ${{ steps.docker_meta.outputs.tags }}
	annotations: ${{ steps.docker_meta.outputs.annotations }}
	labels: ${{ steps.docker_meta.outputs.labels }}
	platforms: linux/amd64,linux/arm64
	- name: Attest
	if: github.event_name == 'push' \|\| github.event_name == 'release'
	uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0
	id: attest
	with:
	subject-name: ghcr.io/${{ github.repository }}
	subject-digest: ${{ steps.docker_push.outputs.digest }}
	push-to-registry: true

	build-aarch64-apple-darwin:
	uses: ./.github/workflows/build-binary.yml
	with:
	target: aarch64-apple-darwin
	os: macos-latest # M-series CPU
	release_mode: ${{ github.event_name != 'pull_request' }}

	build-debian-x86_64:
	uses: ./.github/workflows/build-binary.yml
	with:
	target: debian-x86_64
	# downgraded to have lower libc support. Debian-12 (stable at the time of writing) does not support the same version ubuntu does.
	# TODO: update to ubuntu-latest again once debian 13 launches
	os: ubuntu-22.04
	release_mode: ${{ github.event_name != 'pull_request' }}

	build-x86_64-apple-darwin:
	uses: ./.github/workflows/build-binary.yml
	with:
	target: x86_64-apple-darwin
	os: macos-15-intel # x64 CPU, available until August 2027, see https://github.com/actions/runner-images/issues/13045
	use_cache: ${{ github.event_name != 'release' && github.event_name != 'workflow_dispatch' }}
	release_mode: ${{ github.event_name != 'pull_request' }}

	build-x86_64-pc-windows-msvc:
	uses: ./.github/workflows/build-binary.yml
	with:
	target: x86_64-pc-windows-msvc
	os: windows-latest
	release_mode: ${{ github.event_name != 'pull_request' }}

	build-x86_64-unknown-linux-gnu:
	uses: ./.github/workflows/build-binary.yml
	with:
	target: x86_64-unknown-linux-gnu
	os: ubuntu-latest
	release_mode: ${{ github.event_name != 'pull_request' }}

	build-aarch64-unknown-linux-gnu:
	uses: ./.github/workflows/build-binary.yml
	with:
	target: aarch64-unknown-linux-gnu
	os: ubuntu-24.04-arm
	release_mode: ${{ github.event_name != 'pull_request' }}

	test-aws-lambda:
	name: Test AWS Lambda
	runs-on: ubuntu-latest
	permissions:
	contents: read
	needs: [ build-x86_64-unknown-linux-musl ]
	steps:
	- name: Checkout sources
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with: { persist-credentials: false }
	- uses: taiki-e/install-action@0abfcd587b70a713fdaa7fb502c885e2112acb15 # v2.75.7
	with: { tool: 'just' }
	- name: Download build artifact build-x86_64-unknown-linux-musl
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with: { name: 'build-x86_64-unknown-linux-musl', path: 'target/' }
	- run: just test-lambda target/martin

	test-multi-os:
	name: Test on ${{ matrix.os }}
	runs-on: ${{ matrix.os }}
	permissions:
	contents: read
	needs: [ build-x86_64-pc-windows-msvc, build-x86_64-unknown-linux-gnu, build-debian-x86_64 ]
	env:
	# PG_* variables are used by psql
	PGDATABASE: test
	PGHOST: localhost
	PGUSER: postgres
	PGPASSWORD: postgres
	PGPORT: 34837
	# AWS variables for S3 access
	AWS_REGION: eu-central-1
	AWS_SKIP_CREDENTIALS: 1
	strategy:
	fail-fast: true
	matrix:
	include:
	# temporary disabled due to an issue with homebrew which is difficult to debug
	# - target: x86_64-apple-darwin
	# os: macos-13
	- target: x86_64-pc-windows-msvc
	os: windows-latest
	ext: '.exe'
	- target: x86_64-unknown-linux-gnu
	os: ubuntu-latest
	steps:
	- name: Checkout sources
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with: { persist-credentials: false }
	- name: Set up gdal-bin and sqlite3-tools
	if: matrix.os == 'ubuntu-latest'
	run: sudo apt-get update && sudo apt-get install -y gdal-bin sqlite3-tools
	- name: Install mlt CLI
	if: matrix.os == 'ubuntu-latest'
	uses: taiki-e/install-action@763e3324d4fd026c9bd284c504378585777a87d5 # v2.62.57
	with: { tool: mlt }
	- name: Install sqlite3 and sqldiff on Windows
	if: matrix.os == 'windows-latest'
	run: choco install sqlite --yes
	shell: powershell
	- name: Install and run Postgis (Windows)
	if: matrix.os == 'windows-latest'
	uses: nyurik/action-setup-postgis@e54fc643200ee809b4711c3f537c433dfbeba056 # v2.3
	id: pg
	with: { username: 'postgres', password: 'postgres', database: 'test', postgres-version: 16, port: 34837, postgis_version: 3.5.2 }
	- name: Install and run Postgis (Macos)
	if: matrix.os == 'macos-13'
	uses: nyurik/action-setup-postgis@e54fc643200ee809b4711c3f537c433dfbeba056 # v2.3
	with: { username: 'postgres', password: 'postgres', database: 'test', postgres-version: 16, port: 34837, postgis_version: 3.5.2 }
	- name: Install and run Postgis (Ubuntu)
	if: matrix.os == 'ubuntu-latest'
	run: \|
	docker run -d \
	-p 34837:5432 \
	-e POSTGRES_DB=test \
	-e POSTGRES_USER=postgres \
	-e POSTGRES_PASSWORD=postgres \
	-e PGDATABASE=test \
	-e PGUSER=postgres \
	-e PGPASSWORD=postgres \
	--health-cmd="pg_isready" \
	--health-interval=10s \
	--health-timeout=5s \
	--health-retries=5 \
	postgis/postgis:18-3.6 \
	postgres \
	-c ssl=on \
	-c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem \
	-c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
	- name: Start NGINX
	uses: nyurik/action-setup-nginx@612ee9cb839ad727832cda16359452e7e14dd521 # v1.1
	id: nginx
	with: { port: '5412', output-unix-paths: 'yes' }
	- name: Init database
	env:
	PG_SERVICE_NAME: ${{ steps.pg.outputs.service-name }}
	run: \|
	if [ "$RUNNER_OS" == "Linux" ]; then
	export DATABASE_URL="postgres://${PGUSER}:${PGUSER}@${PGHOST}:${PGPORT}/${PGDATABASE}?sslmode=require"
	else
	export PGSERVICE="${PG_SERVICE_NAME}"
	fi
	tests/fixtures/initdb.sh
	- name: Copy static files
	env:
	HTML_DIR: ${{ steps.nginx.outputs.html-dir }}
	run: cp -r tests/fixtures/pmtiles2/* "$HTML_DIR"
	- name: Download build artifact build-${{ matrix.target }}
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: build-${{ matrix.target }}
	path: target/

	- name: Integration Tests
	run: \|
	export MARTIN_BUILD_ALL=-
	export AWS_REGION=eu-central-1
	export AWS_SKIP_CREDENTIALS=1
	export MARTIN_BIN=target/martin${{ matrix.ext }}
	export MARTIN_CP_BIN=target/martin-cp${{ matrix.ext }}
	export MBTILES_BIN=target/mbtiles${{ matrix.ext }}
	if [[ "${{ runner.os }}" != "Windows" ]]; then
	chmod +x "$MARTIN_BIN" "$MARTIN_CP_BIN" "$MBTILES_BIN"
	fi
	if [ "$RUNNER_OS" == "Linux" ]; then
	export DATABASE_URL="postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ env.PGPORT }}/${{ env.PGDATABASE }}?sslmode=require"
	else
	export DATABASE_URL="${STEPS_PG_OUTPUTS_CONNECTION_URI}"
	fi
	tests/test.sh
	env:
	STEPS_PG_OUTPUTS_CONNECTION_URI: ${{ steps.pg.outputs.connection-uri }}
	- name: Compare test output results (Linux)
	if: matrix.target == 'x86_64-unknown-linux-gnu'
	run: \|
	if ! diff --brief --recursive --new-file --exclude='*.pbf' tests/output tests/expected; then
	echo "** Expected output does not match actual output"
	echo "** If this is expected, run 'just bless' to update expected output"
	echo ""
	echo "::group::Resulting diff (max 100 lines)"
	diff --recursive --new-file --exclude='*.pbf' tests/output tests/expected \| head -n 100 \| cat --show-nonprinting
	echo "::endgroup::"
	exit 1
	else
	echo "** Expected output matches actual output"
	fi
	- name: Download Debian package (Linux)
	if: matrix.target == 'x86_64-unknown-linux-gnu'
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with: { name: 'build-debian-x86_64', path: 'target/' }
	- name: Tests Debian package (Linux)
	if: matrix.target == 'x86_64-unknown-linux-gnu'
	run: \|
	sudo dpkg -i target/debian-x86_64.deb
	export MARTIN_BUILD_ALL=-
	export AWS_REGION=eu-central-1
	export AWS_SKIP_CREDENTIALS=1
	export MARTIN_BIN=/usr/bin/martin${{ matrix.ext }}
	export MARTIN_CP_BIN=/usr/bin/martin-cp${{ matrix.ext }}
	export MBTILES_BIN=/usr/bin/mbtiles${{ matrix.ext }}
	tests/test.sh
	env:
	DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ env.PGPORT }}/${{ env.PGDATABASE }}?sslmode=require
	- name: Save test output (on error)
	if: failure()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: failed-test-output-${{ runner.os }}
	path: \|
	tests/output/*
	target/test_logs/*
	retention-days: 5

	test-freebsd:
	name: Unit test on FreeBSD
	runs-on: ubuntu-latest
	permissions:
	contents: read
	needs:
	- lint-rust
	steps:
	- name: Checkout sources
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with: { persist-credentials: false }
	- name: Test in FreeBSD
	id: test
	uses: vmactions/freebsd-vm@7ca82f79fe3078fecded6d3a2bff094995447bbd # v1
	with:
	release: "15.0"
	envs: "AWS_SKIP_CREDENTIALS AWS_REGION"
	# Dependencies (FreeBSD base systems come more bare):
	# - sqlite3 for MBTiles support
	# - freetype2 for font rendering to SDF
	# - cmake for aws-lc-sys
	# - just for running standardized test suites
	# - bash because the justfile has a lot of scripts that assume bash
	# - node and npm are not currently used (we don't run the JS tests on FreeBSD),
	# but they are here in case we want them later.
	prepare: \|
	pkg install -y rust sqlite3 freetype2 cmake just bash node npm
	run: \|
	just env-info
	just test-freebsd
	env:
	AWS_SKIP_CREDENTIALS: 1
	AWS_REGION: eu-central-1

	unit-test-svc-pg:
	name: Unit test postgis:${{ matrix.img_ver }} sslmode=${{ matrix.sslmode }}
	runs-on: ubuntu-latest
	permissions:
	contents: read
	strategy: &svc-strategy
	fail-fast: true
	matrix:
	include:
	# These must match the versions of postgres used in the docker-compose.yml
	- img_ver: 11-3.0-alpine
	args: postgres
	sslmode: disable
	- img_ver: 18-3.6-alpine
	args: postgres
	sslmode: disable
	# alpine images don't support SSL, so for this we use the debian images
	- img_ver: 18-3.6
	args: postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
	sslmode: require
	#
	# FIXME!
	# DISABLED because Rustls fails to validate name (CN?) with the NotValidForName error
	#- img_ver: 15-3.3
	# args: postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
	# sslmode: verify-ca
	#- img_ver: 15-3.3
	# args: postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
	# sslmode: verify-full
	env: &svc-env
	PGDATABASE: test
	PGHOST: localhost
	PGUSER: postgres
	PGPASSWORD: postgres
	services:
	postgres: &svc-postgres
	image: postgis/postgis:${{ matrix.img_ver }}
	ports:
	- 5432/tcp
	options: >-
	-e POSTGRES_DB=test
	-e POSTGRES_USER=postgres
	-e POSTGRES_PASSWORD=postgres
	-e PGDATABASE=test
	-e PGUSER=postgres
	-e PGPASSWORD=postgres
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	--entrypoint sh
	postgis/postgis:${{ matrix.img_ver }}
	-c "exec docker-entrypoint.sh ${{ matrix.args }}"
	steps:
	- run: rustup update stable && rustup default stable
	- name: Checkout sources
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with: { persist-credentials: false }
	- uses: taiki-e/install-action@0abfcd587b70a713fdaa7fb502c885e2112acb15 # v2.75.7
	with: { tool: 'just' }
	- name: Init database
	run: tests/fixtures/initdb.sh
	env:
	PGPORT: ${{ job.services.postgres.ports[5432] }}
	- name: Unit Tests
	if: matrix.sslmode != 'verify-ca' && matrix.sslmode != 'verify-full'
	run: \|
	echo "Running unit tests, connecting to DATABASE_URL=$DATABASE_URL"
	echo "Same but as base64 to prevent GitHub obfuscation (this is not a secret):"
	echo "$DATABASE_URL" \| base64
	just test-pg
	env:
	DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }}

	test-with-svc:
	name: Test postgis:${{ matrix.img_ver }} sslmode=${{ matrix.sslmode }}
	runs-on: ubuntu-latest
	permissions:
	contents: read
	needs: [ build-x86_64-unknown-linux-gnu, build-debian-x86_64, unit-test-svc-pg ]
	strategy: *svc-strategy
	env: *svc-env
	services:
	postgres: *svc-postgres
	steps:
	- name: Checkout sources
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with: { persist-credentials: false }
	- name: Run NGINX
	uses: nyurik/action-setup-nginx@612ee9cb839ad727832cda16359452e7e14dd521 # v1.1
	id: nginx
	with: { port: '5412', output-unix-paths: 'yes' }
	- name: Copy static files
	env:
	HTML_DIR: ${{ steps.nginx.outputs.html-dir }}
	run: cp -r tests/fixtures/pmtiles2/* "$HTML_DIR"
	- name: Init database
	run: tests/fixtures/initdb.sh
	env:
	PGPORT: ${{ job.services.postgres.ports[5432] }}
	- name: Get DB SSL cert (sslmode=verify-*)
	if: matrix.sslmode == 'verify-ca' \|\| matrix.sslmode == 'verify-full'
	run: \|
	set -x
	mkdir -p target/certs
	docker cp ${JOB_SERVICES_POSTGRES_ID}:/etc/ssl/certs/ssl-cert-snakeoil.pem target/certs/server.crt
	docker cp ${JOB_SERVICES_POSTGRES_ID}:/etc/ssl/private/ssl-cert-snakeoil.key target/certs/server.key
	env:
	JOB_SERVICES_POSTGRES_ID: ${{ job.services.postgres.id }}
	- name: Download build artifact build-x86_64-unknown-linux-gnu
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with: { name: 'build-x86_64-unknown-linux-gnu', path: 'target_releases/' }
	- name: Set up gdal-bin and sqlite3-tools
	run: sudo apt update && sudo apt-get install -y gdal-bin sqlite3-tools
	- name: Integration Tests
	run: \|
	if [[ "${{ matrix.sslmode }}" == "verify-ca" \|\| "${{ matrix.sslmode }}" == "verify-full" ]]; then
	export PGSSLROOTCERT=target/certs/server.crt
	fi
	export MARTIN_BUILD_ALL=-
	export AWS_REGION=eu-central-1
	export AWS_SKIP_CREDENTIALS=1
	export MARTIN_BIN=target_releases/martin
	export MARTIN_CP_BIN=target_releases/martin-cp
	export MBTILES_BIN=target_releases/mbtiles
	chmod +x "$MARTIN_BIN" "$MARTIN_CP_BIN" "$MBTILES_BIN"
	tests/test.sh
	rm -rf target_releases
	env:
	DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }}
	- name: Download Debian package
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with: { name: 'build-debian-x86_64', path: 'target_releases/' }
	- name: Tests Debian package
	run: \|
	sudo dpkg -i target_releases/debian-x86_64.deb
	if [[ "${{ matrix.sslmode }}" == "verify-ca" \|\| "${{ matrix.sslmode }}" == "verify-full" ]]; then
	export PGSSLROOTCERT=target/certs/server.crt
	fi
	export MARTIN_BUILD_ALL=-
	export AWS_REGION=eu-central-1
	export AWS_SKIP_CREDENTIALS=1
	export MARTIN_BIN=/usr/bin/martin
	export MARTIN_CP_BIN=/usr/bin/martin-cp
	export MBTILES_BIN=/usr/bin/mbtiles
	tests/test.sh
	sudo dpkg -P martin
	rm -rf target_releases
	env:
	DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }}
	- name: Save test output (on error)
	if: failure()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: test-output
	path: \|
	tests/output/*
	target/test_logs/*
	retention-days: 5

	test-publish:
	name: Test publishing via cargo-publish
	permissions:
	contents: read
	runs-on: ubuntu-latest
	steps:
	- run: rustup update stable && rustup default stable
	- name: Checkout sources
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with: { persist-credentials: false }
	- name: Test packaging
	run: cargo publish --workspace --dry-run

	package:
	name: Package
	runs-on: ubuntu-latest
	permissions:
	# for writing attestations
	id-token: write
	attestations: write
	# for writing release artifacts
	contents: write
	needs: [ lint-unit-test-js, unit-test-rust, lint-rust, lint-rust-dependencies, test-multi-os, test-with-svc, test-aws-lambda, test-freebsd, test-publish, build-aarch64-apple-darwin, build-x86_64-apple-darwin, build-aarch64-unknown-linux-gnu, build-aarch64-unknown-linux-musl ]
	steps:
	- name: Checkout sources
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with: { persist-credentials: false }
	- uses: taiki-e/install-action@0abfcd587b70a713fdaa7fb502c885e2112acb15 # v2.75.7
	with: { tool: 'just' }
	- name: Download build artifact build-aarch64-apple-darwin
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with: { name: 'build-aarch64-apple-darwin', path: 'target/aarch64-apple-darwin' }
	- name: Download build artifact build-x86_64-apple-darwin
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with: { name: 'build-x86_64-apple-darwin', path: 'target/x86_64-apple-darwin' }
	- name: Download build artifact build-aarch64-unknown-linux-gnu
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with: { name: 'build-aarch64-unknown-linux-gnu', path: 'target/aarch64-unknown-linux-gnu' }
	- name: Download build artifact build-x86_64-unknown-linux-gnu
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with: { name: 'build-x86_64-unknown-linux-gnu', path: 'target/x86_64-unknown-linux-gnu' }
	- name: Download build artifact build-aarch64-unknown-linux-musl
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with: { name: 'build-aarch64-unknown-linux-musl', path: 'target/aarch64-unknown-linux-musl' }
	- name: Download build artifact build-x86_64-unknown-linux-musl
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with: { name: 'build-x86_64-unknown-linux-musl', path: 'target/x86_64-unknown-linux-musl' }
	- name: Download build artifact build-x86_64-pc-windows-msvc
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with: { name: 'build-x86_64-pc-windows-msvc', path: 'target/x86_64-pc-windows-msvc' }
	- name: Download build artifact build-debian-x86_64
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with: { name: 'build-debian-x86_64', path: 'target/debian-x86_64' }

	- name: Package
	run: \|
	set -x

	for target in aarch64-apple-darwin x86_64-apple-darwin aarch64-unknown-linux-gnu x86_64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl x86_64-pc-windows-msvc debian-x86_64;
	do
	just package-assets "$target"
	done

	- name: Create Homebrew config
	run: \|
	set -x

	# Extract Github release version only without the "martin-v" prefix from release tag names (martin-vX.Y.Z)
	# On non-release events, this still generates a config with not-a-release as the version to test the release process
	MARTIN_VERSION=$(echo "$GITHUB_EVENT_RELEASE_TAG_NAME" \| sed -e 's/martin-v//')

	mkdir -p target/homebrew
	cd target

	cat << EOF > homebrew_config.yaml
	version: "$MARTIN_VERSION"
	macos_arm_sha256: "$(shasum -a 256 files/martin-aarch64-apple-darwin.tar.gz \| cut -d' ' -f1)"
	macos_intel_sha256: "$(shasum -a 256 files/martin-x86_64-apple-darwin.tar.gz \| cut -d' ' -f1)"
	linux_arm_sha256: "$(shasum -a 256 files/martin-aarch64-unknown-linux-musl.tar.gz \| cut -d' ' -f1)"
	linux_intel_sha256: "$(shasum -a 256 files/martin-x86_64-unknown-linux-musl.tar.gz \| cut -d' ' -f1)"
	EOF
	env:
	GITHUB_EVENT_RELEASE_TAG_NAME: ${{ github.event.release.tag_name \|\| 'not-a-release' }}

	- name: Save Homebrew Config
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with: { name: 'homebrew-config', path: 'target/homebrew_config.yaml' }

	- name: Generate SLSA build provenance attestation
	if: github.event_name == 'release'
	uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0
	with: { subject-path: 'target/files/*' }
	- name: Publish with artifacts
	if: ${{ github.event_name == 'release' && startsWith(github.event.release.tag_name, 'martin-v') }}
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	RELEASE_TAG_NAME: ${{ github.event.release.tag_name }}
	run: \|
	set -euo pipefail
	if ! compgen -G "target/files/*" > /dev/null; then
	echo "No release artifacts matched target/files/*" >&2
	exit 1
	fi
	gh release upload "$RELEASE_TAG_NAME" target/files/* --clobber
	gh release edit "$RELEASE_TAG_NAME" --verify-tag --latest
	- name: Publish without artifacts
	if: ${{ github.event_name == 'release' && !startsWith(github.event.release.tag_name, 'martin-v') }}
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	RELEASE_TAG_NAME: ${{ github.event.release.tag_name }}
	run: gh release edit "$RELEASE_TAG_NAME" --verify-tag --latest=false

	# Create a personal access token
	# Gen: https://github.com/settings/personal-access-tokens/new
	# Can also use "regenerate token" button in the expired token screen
	# Set: https://github.com/maplibre/martin/settings/secrets/actions/GH_HOMEBREW_MARTIN_TOKEN
	# Docs: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token
	# Name: anything descriptive
	# One year long (sadly that's max)
	# Repository owner and repo: maplibre/homebrew-martin
	# Access Contents: Read and write
	# Access Metadata: Read-only
	# Access Pull requests: Read and write
	- name: Checkout maplibre/homebrew-martin
	if: github.event_name == 'release'
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	repository: maplibre/homebrew-martin
	token: ${{ secrets.GH_HOMEBREW_MARTIN_TOKEN }} # See instructions above
	path: target/homebrew
	persist-credentials: false

	- name: Create Homebrew formula
	uses: cuchi/jinja2-action@8bd801365a8d36a0b20f45ee969161685de7785a # master
	with:
	template: .github/files/homebrew.martin.rb.j2
	output_file: target/homebrew/martin.rb
	data_file: target/homebrew_config.yaml

	- name: Create a PR for maplibre/homebrew-martin
	if: github.event_name == 'release' && startsWith(github.event.release.tag_name, 'martin-v')
	uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
	with:
	token: ${{ secrets.GH_HOMEBREW_MARTIN_TOKEN }} # See instructions above
	commit-message: 'Update to ${{ github.event.release.tag_name }}'
	title: 'Update to ${{ github.event.release.tag_name }}'
	body: 'Update to ${{ github.event.release.tag_name }}'
	branch: 'update-to-${{ github.event.release.tag_name }}'
	branch-suffix: timestamp
	base: 'main'
	labels: 'auto-update'
	assignees: 'nyurik'
	draft: false
	delete-branch: true
	path: target/homebrew

	# This final step is needed to mark the whole workflow as successful
	# Don't change its name - it is used by the merge protection rules
	done:
	name: CI Finished
	runs-on: ubuntu-latest
	needs:
	- package
	- docker-build-test
	- unit-test-rust
	- lint-rust-dependencies
	- lint-unit-test-js
	- lint-rust
	# - test-freebsd Fixme, fails
	if: always()
	permissions: {}
	steps:
	- name: Result of the needed steps
	run: echo "${{ toJSON(needs) }}" # zizmor: ignore[template-injection]
	- if: ${{ contains(needs..result, 'failure') \|\| contains(needs..result, 'cancelled') \|\| contains(needs.*.result, 'skipped') }}
	name: CI Result
	run: exit 1

	# Release unpublished packages or create a PR with changes
	release-plz:
	needs: [ done ]
	if: \|
	always()
	&& needs.done.result == 'success'
	&& github.event_name == 'push'
	&& github.ref == 'refs/heads/main'
	&& github.repository_owner == 'maplibre'
	runs-on: ubuntu-latest
	permissions:
	contents: read
	id-token: write
	concurrency:
	group: release-plz-${{ github.ref }}
	cancel-in-progress: false
	steps:
	- run: rustup update stable && rustup default stable
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with: { fetch-depth: 0, persist-credentials: false }
	- name: Publish to crates.io if crate's version is newer
	uses: release-plz/action@1528104d2ca23787631a1c1f022abb64b34c1e11 # v0.5.128
	id: release
	with: { command: release }
	env:
	GITHUB_TOKEN: ${{ secrets.RELEASE_PLZ_TOKEN }}
	- if: ${{ steps.release.outputs.releases_created == 'false' }}
	name: If version is the same, create a PR proposing new version and changelog for the next release
	uses: release-plz/action@1528104d2ca23787631a1c1f022abb64b34c1e11 # v0.5.128
	with: { command: release-pr }
	env:
	GITHUB_TOKEN: ${{ secrets.RELEASE_PLZ_TOKEN }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

feat: impl pre-processing encodings #8740

Workflow file

feat: impl pre-processing encodings #8740

Uh oh!

Workflow file for this run