Skip to content

Commit 4910a62

Browse files
MaxRev-DevMaxRev-Dev
authored
Feature: Moving dockerized Linux CI to native (#205)
* Moving dockerized Linux CI to native * Change script permissions.
1 parent fdf4bf1 commit 4910a62

File tree

5 files changed

+101
-113
lines changed

5 files changed

+101
-113
lines changed

‎.github/workflows/macos.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -89,15 +89,15 @@ jobs:
8989
9090
- name: Compile HDF
9191
run: |
92-
make -f gdal-makefile hdf-force BUILD_ARCH=${{ matrix.arch }}
92+
make -f gdal-makefile hdf BUILD_ARCH=${{ matrix.arch }}
9393
9494
- name: Compile PROJ
9595
run: |
96-
make -f gdal-makefile proj-force BUILD_ARCH=${{ matrix.arch }}
96+
make -f gdal-makefile proj BUILD_ARCH=${{ matrix.arch }}
9797
9898
- name: Compile GDAL
9999
run: |
100-
make -f gdal-makefile gdal-force BUILD_ARCH=${{ matrix.arch }}
100+
make -f gdal-makefile gdal BUILD_ARCH=${{ matrix.arch }}
101101
102102
- name: Collect deps
103103
run: |

‎.github/workflows/unix.yml‎

Lines changed: 75 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,8 @@ on:
1717
required: true
1818

1919
env:
20-
DOCKER_REGISTRY: ghcr.io/maxrev-dev
21-
22-
permissions:
23-
packages: write
24-
contents: read
20+
DOTNET_INSTALL_DIR: ${{ github.workspace }}/.dotnet/
21+
VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg-archives/
2522

2623
jobs:
2724
BuildNugetPackages-Linux:
@@ -37,6 +34,9 @@ jobs:
3734
actions-runner: ubuntu-24.04-arm
3835

3936
runs-on: ${{ matrix.actions-runner }}
37+
defaults:
38+
run:
39+
working-directory: unix
4040

4141
steps:
4242
- name: Checkout repository
@@ -50,81 +50,59 @@ jobs:
5050
api-user-github: ${{ secrets.API_USER_GITHUB }}
5151
api-user-nuget: ${{ secrets.API_USER_NUGET }}
5252

53-
- name: Set up QEMU
54-
uses: docker/[email protected]
55-
56-
- name: Set up Docker Buildx
57-
uses: docker/[email protected]
58-
59-
- name: Cache restore
60-
uses: actions/[email protected]
53+
- name: Restore cached packages
54+
id: build-packages-cache-restore
55+
uses: actions/cache@v4
6156
with:
6257
path: |
6358
.dotnet/
64-
ci/cache/
65-
key: ${{ matrix.os }}-buildx-${{ matrix.arch }}-${{ github.run_id }}
59+
vcpkg-archives/
60+
key: ${{ matrix.os }}-packages-${{ matrix.arch }}-${{ github.run_id }}
6661
restore-keys: |
67-
${{ matrix.os }}-buildx-${{ matrix.arch }}-
68-
69-
- name: Ensure cache directories
62+
${{ matrix.os }}-packages-${{ matrix.arch }}-
63+
64+
- name: Install CMake
7065
run: |
71-
mkdir -p ci/cache/.dotnet
72-
mkdir -p ci/cache/vcpkg-archives
66+
pipx install cmake
7367
74-
- name: Log in to the Container registry
75-
uses: docker/login-action@v3
76-
with:
77-
registry: ghcr.io
78-
username: ${{ github.actor }}
79-
password: ${{ secrets.GITHUB_TOKEN }}
80-
logout: false
81-
82-
- name: Extract metadata (tags, labels) for Docker
83-
id: meta
84-
uses: docker/metadata-action@v5
85-
with:
86-
images: |
87-
${{ env.DOCKER_REGISTRY }}/gdal.netcore.builder.${{ matrix.arch }}
88-
tags: |
89-
type=sha
90-
type=ref,event=branch
91-
type=ref,event=pr
92-
latest
93-
94-
- name: Build and push on GH runner
95-
uses: docker/build-push-action@v5
96-
if: ${{ !contains(matrix.actions-runner, 'self-hosted') }}
97-
with:
98-
push: true
99-
context: ${{ github.workspace }}
100-
file: ci/Dockerfile.unix
101-
platforms: ${{ matrix.os }}
102-
build-args: |
103-
BUILD_NUMBER_TAIL=${{ github.run_number }}
104-
PRERELEASE=${{ inputs.is-pre-release }}
105-
tags: ${{ steps.meta.outputs.tags }}
106-
labels: ${{ steps.meta.outputs.labels }}
107-
108-
- name: Build and push on local runner
109-
uses: docker/build-push-action@v5
110-
if: ${{ contains(matrix.actions-runner, 'self-hosted') }}
68+
- name: Setup .NET Core SDK
69+
uses: actions/[email protected]
11170
with:
112-
load: true
113-
push: false
114-
tags: ${{ env.DOCKER_REGISTRY }}/gdal.netcore.builder.${{ matrix.arch }}:latest
115-
context: .
116-
file: ci/Dockerfile.unix
117-
platforms: ${{ matrix.os }}
118-
build-args: |
119-
BUILD_NUMBER_TAIL=${{ github.run_number }}
120-
PRERELEASE=${{ inputs.is-pre-release }}
121-
DOTNET_INSTALL_DIR=/build/ci/cache/.dotnet
122-
VCPKG_DEFAULT_BINARY_CACHE=/build/ci/cache/vcpkg-archives/
123-
124-
- name: Extract artifacts
71+
dotnet-version: ${{ inputs.dotnet-version }}
72+
73+
- name: Install deps
74+
run: |
75+
chmod +x ${{ github.workspace }}/unix/before-install.sh
76+
${{ github.workspace }}/unix/before-install.sh
77+
78+
- name: Fetch VCPKG packages
12579
run: |
126-
chmod +x '${{ github.workspace }}/ci/extract-container-artifacts.sh'
127-
'${{ github.workspace }}/ci/extract-container-artifacts.sh' ${{ matrix.arch }}
80+
mkdir -p ${{ github.workspace }}/vcpkg-archives/
81+
make -f vcpkg-makefile BUILD_ARCH=${{ matrix.arch }}
82+
83+
- name: Init GDAL and PROJ
84+
run: |
85+
make -f gdal-makefile init_gdal init_proj BUILD_ARCH=${{ matrix.arch }}
86+
87+
- name: Compile HDF
88+
run: |
89+
make -f gdal-makefile hdf BUILD_ARCH=${{ matrix.arch }}
90+
91+
- name: Compile PROJ
92+
run: |
93+
make -f gdal-makefile proj BUILD_ARCH=${{ matrix.arch }}
94+
95+
- name: Compile GDAL
96+
run: |
97+
make -f gdal-makefile gdal BUILD_ARCH=${{ matrix.arch }} DOTNET_VERSION=${{ inputs.dotnet-version }}
98+
99+
- name: Collect deps
100+
run: |
101+
make -f collect-deps-makefile BUILD_ARCH=${{ matrix.arch }}
102+
103+
- name: Create packages
104+
run: |
105+
make -f publish-makefile pack BUILD_ARCH=${{ matrix.arch }} BUILD_NUMBER_TAIL=${{ github.run_number }}
128106
129107
- name: Store metadata as artifact
130108
uses: actions/[email protected]
@@ -142,6 +120,7 @@ jobs:
142120
path: |
143121
tests/gdal-formats/*.txt
144122
overwrite: true
123+
145124
- name: Store runtime packages as artifact
146125
uses: actions/[email protected]
147126
with:
@@ -159,8 +138,8 @@ jobs:
159138
path: nuget/MaxRev.Gdal.Core.*.nupkg
160139
overwrite: true
161140

141+
162142
TestPackages-Linux:
163-
needs: BuildNugetPackages-Linux
164143
strategy:
165144
matrix:
166145
include:
@@ -171,6 +150,7 @@ jobs:
171150
arch: arm64
172151
actions-runner: ubuntu-24.04-arm
173152

153+
needs: BuildNugetPackages-Linux
174154
runs-on: ${{ matrix.actions-runner }}
175155
defaults:
176156
run:
@@ -185,12 +165,28 @@ jobs:
185165
with:
186166
name: packages-core
187167
path: nuget/
168+
merge-multiple: true
188169

189170
- name: Download artifact - runtime packages
190171
uses: actions/[email protected]
191172
with:
192173
name: packages-unix-${{ matrix.arch }}
193174
path: nuget/
175+
merge-multiple: true
176+
177+
178+
- name: Setup .NET Core SDK
179+
uses: actions/[email protected]
180+
with:
181+
dotnet-version: ${{ inputs.dotnet-version }}
182+
183+
- name: NuGet - set credentials
184+
uses: ./.github/actions/set-credentials
185+
with:
186+
api-key-github: ${{ secrets.API_KEY_GITHUB }}
187+
api-key-nuget: ${{ secrets.API_KEY_NUGET }}
188+
api-user-github: ${{ secrets.API_USER_GITHUB }}
189+
api-user-nuget: ${{ secrets.API_USER_NUGET }}
194190

195191
- name: Remove old formats
196192
run: |
@@ -203,54 +199,32 @@ jobs:
203199
path: "tests/gdal-formats"
204200
merge-multiple: true
205201

206-
- name: Set up Docker Buildx
207-
uses: docker/[email protected]
208-
209-
- name: Build docker image
210-
run: |
211-
docker buildx build --load --platform=${{ matrix.os }} \
212-
--progress=plain -t maxrev-dev/gdal.netcore.tester.${{ matrix.arch }}:latest \
213-
--build-arg BUILD_NUMBER_TAIL=${{ github.run_number }} \
214-
--build-arg BUILD_ARCH=${{ matrix.arch }} \
215-
--build-arg PRERELEASE=${{ inputs.is-pre-release }} \
216-
-f '${{ github.workspace }}/ci/Dockerfile.unix.test' \
217-
${{ github.workspace }}
218-
219202
- name: Test packages
220203
run: |
221-
docker run --rm \
222-
--env BUILD_NUMBER_TAIL=${{ github.run_number }} \
223-
--env BUILD_ARCH=${{ matrix.arch }} \
224-
--env PRERELEASE=${{ inputs.is-pre-release }} \
225-
-t maxrev-dev/gdal.netcore.tester.${{ matrix.arch }}:latest
204+
make -f test-makefile BUILD_ARCH=${{ matrix.arch }} BUILD_NUMBER_TAIL=${{ github.run_number }}
226205
227206
PushPackages-Linux:
228207
needs: TestPackages-Linux
229208
runs-on: ubuntu-latest
230209
defaults:
231210
run:
232211
working-directory: unix
233-
234212
steps:
235213
- name: Checkout repository
236214
uses: actions/[email protected]
237215

238-
- name: Setup .NET Core SDK
239-
uses: actions/[email protected]
240-
with:
241-
dotnet-version: ${{ inputs.dotnet-version }}
242-
243216
- name: Download artifact - core package
244217
uses: actions/[email protected]
245218
with:
246-
pattern: packages-core
247-
path: "${{ github.workspace}}/nuget/"
219+
name: packages-core
220+
path: nuget/
221+
merge-multiple: true
248222

249223
- name: Download artifact - runtime packages
250224
uses: actions/[email protected]
251225
with:
252226
pattern: packages-unix*
253-
path: "${{ github.workspace}}/nuget/"
227+
path: nuget/
254228
merge-multiple: true
255229

256230
- name: Download artifact - metadata
@@ -281,5 +255,5 @@ jobs:
281255
- name: Push packages
282256
if: ${{ github.event.pull_request.merged == true || github.ref == 'refs/heads/main' || fromJson(inputs.is-version-branch) }}
283257
run: |
284-
make -f push-packages-makefile PRERELEASE=${{ inputs.is-pre-release }} INCLUDE_CORE=1 \
258+
make -f ../unix/push-packages-makefile PRERELEASE=${{ inputs.is-pre-release }} INCLUDE_CORE=1 \
285259
BUILD_NUMBER_TAIL=${{ github.run_number }} API_KEY_GITHUB=${{ secrets.API_KEY_GITHUB }} API_KEY_NUGET=${{ secrets.API_KEY_NUGET }}

‎osx/gdal-makefile‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,13 +79,15 @@ init_hdf: check_hdf_sources
7979
@echo "$(TO) HDF sources restore complete"
8080

8181
HDF_zip=hdf$(HDF_VERSION).zip
82-
HDF_SOURCE=$(BUILD_ROOT)/hdf-$(HDF_VERSION)/hdfsrc
82+
HDF_SOURCE=$(BUILD_ROOT)/hdf4-hdf$(HDF_VERSION)
8383

8484
download_hdf:
85-
@echo "$(TO) Downloading HDF sources..."
86-
curl -L -o "$(BUILD_ROOT)/$(HDF_zip)" "https://github.com/HDFGroup/hdf4/releases/download/hdf$(HDF_VERSION)/$(HDF_zip)"
87-
cd "$(BUILD_ROOT)"; unzip -q "$(BUILD_ROOT)/$(HDF_zip)" -d "$(BUILD_ROOT)/hdf-$(HDF_VERSION)"
88-
@echo "$(TO) HDF sources downloaded!"
85+
@echo "$(TO) Downloading HDF source ${HDF_zip}..."
86+
LD_LIBRARY_PATH="" curl -JL "https://github.com/HDFGroup/hdf4/archive/refs/tags/$(HDF_zip)" -o "$(BUILD_ROOT)/$(HDF_zip)"
87+
@echo "$(TO) HDF source downloaded!"
88+
@echo "$(TO) Extracting HDF source..."
89+
cd "$(BUILD_ROOT)"; unzip -oq "$(BUILD_ROOT)/$(HDF_zip)" -d .
90+
8991

9092
check_hdf_sources:
9193
@if [[ ! -f "$(BUILD_ROOT)/$(HDF_zip)" ]] || [[ ! -d "$(HDF_SOURCE)" ]]; then \

‎unix/before-install.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
set -e
3+
sudo apt-get update
4+
sudo apt-get install g++ make cmake git curl \
5+
zip unzip tar pkg-config linux-headers-generic libltdl-dev \
6+
autoconf automake python3 autoconf-archive swig patchelf -y

‎unix/gdal-makefile‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -76,12 +76,18 @@ remove_cache_%:
7676
rm -r $($(UP)_CMAKE_TMP); \
7777
fi;
7878

79-
HDF_zip=CMake-hdf-$(HDF_VERSION).zip
80-
HDF_SOURCE=$(BUILD_ROOT)/CMake-hdf-$(HDF_VERSION)/hdf-$(HDF_VERSION)
79+
init_hdf: check_hdf_sources
80+
@echo "$(TO) HDF sources restore complete"
81+
82+
HDF_zip=hdf$(HDF_VERSION).zip
83+
HDF_SOURCE=$(BUILD_ROOT)/hdf4-hdf$(HDF_VERSION)
8184

8285
download_hdf:
83-
curl -o "$(BUILD_ROOT)/$(HDF_zip)" "https://hdf-wordpress-1.s3.amazonaws.com/wp-content/uploads/manual/HDF4/HDF$(HDF_VERSION)/src/$(HDF_zip)"
84-
cd "$(BUILD_ROOT)"; unzip -q "$(BUILD_ROOT)/$(HDF_zip)" -d .
86+
@echo "$(TO) Downloading HDF source ${HDF_zip}..."
87+
LD_LIBRARY_PATH="" curl -JL "https://github.com/HDFGroup/hdf4/archive/refs/tags/$(HDF_zip)" -o "$(BUILD_ROOT)/$(HDF_zip)"
88+
@echo "$(TO) HDF source downloaded!"
89+
@echo "$(TO) Extracting HDF source..."
90+
cd "$(BUILD_ROOT)"; unzip -oq "$(BUILD_ROOT)/$(HDF_zip)" -d .
8591

8692
check_hdf_sources:
8793
@if [[ ! -f "$(BUILD_ROOT)/$(HDF_zip)" ]] || [[ ! -d "$(HDF_SOURCE)" ]]; then \

0 commit comments

Comments
 (0)