upload artifacts (#124)

isaacbrodsky · web-flow · commit 321a3f57be8b · 2023-01-30T14:57:30.000-08:00
Change the build process to upload native shared objects during the Github Actions process, and to add an option for downloading them again when building the library (rather than building them locally.) This will be used to release the library from machines that do not support the exact cross-compilation strategies used in Github Actions workflows.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -45,6 +45,16 @@ jobs:
       - name: Format check for C
         run: git diff --exit-code
 
+      - uses: actions/upload-artifact@v3
+        name: Upload artifacts
+        if: ${{ matrix.java-version == 8 }}
+        with:
+          name: docker-built-shared-objects
+          path: |
+            src/main/resources/*/*.so
+            src/main/resources/*/*.dll
+          if-no-files-found: error 
+
   tests-new-dockcross:
     name: Dockcross ${{ matrix.dockcross-tag }} Java ${{ matrix.java-version }} ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
@@ -54,7 +64,7 @@ jobs:
         os: [ubuntu-latest]
         java-distribution: [adopt]
         java-version: [17]
-        dockcross-tag: ["20220906-e88a3ce"]
+        dockcross-tag: ["20220906-e88a3ce", "20230116-670f7f7"]
 
     steps:
       - uses: actions/checkout@v2.1.1
@@ -109,6 +119,14 @@ jobs:
       - name: Tests
         run: mvn -B -V clean test site
 
+      - uses: actions/upload-artifact@v3
+        name: Upload Mac OS Artifacts
+        if: ${{ matrix.os == 'macos-latest' && matrix.java-version == 8 }}
+        with:
+          name: macos-built-shared-objects
+          path: src/main/resources/*/*.dylib
+          if-no-files-found: error 
+
   tests-coverage:
     name: Java (Coverage) ${{ matrix.java-version }} ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
@@ -143,3 +161,52 @@ jobs:
         env:
           CI_NAME: github
           COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }} 
+
+  tests-use-built-artifacts:
+    name: Java (Built Artifacts) ${{ matrix.java-version }} ${{ matrix.os }}
+    needs:
+      - tests
+      - tests-no-docker
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        java-distribution: [adopt]
+        java-version: [8]
+
+    steps:
+      - uses: actions/checkout@v2.1.1
+        with:
+          submodules: recursive
+
+      - uses: actions/setup-java@v2
+        with:
+          distribution: "${{ matrix.java-distribution }}"
+          java-version: "${{ matrix.java-version }}"
+
+      - uses: actions/cache@v2
+        id: maven-cache
+        with:
+          path: ~/.m2/
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+
+      - name: Download Docker binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: docker-built-shared-objects
+          path: src/main/resources/
+
+      - name: Download Mac binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: macos-built-shared-objects
+          path: src/main/resources/
+
+      - name: Download and test
+        run: |
+          mvn clean test -Dh3.github.artifacts.use=true -Dh3.github.artifacts.by_run=true
+        env:
+          GH_TOKEN: ${{ github.token }}
diff --git a/docs/releasing.md b/docs/releasing.md
@@ -8,13 +8,18 @@ Release builds should only be run on Mac OSX with Docker. This is needed so that
 
 1. Ensure you are on branch `master` and that the library is building correctly.
 2. Update [CHANGELOG.md](../CHANGELOG.md) to have the correct date and new version number, update [README.md](../README.md) to have the correct version numbers, and commit.
-3. `mvn release:prepare` Use the new version number when prompted.
-4. `mvn release:perform`
+3. `mvn release:prepare -Dh3.github.artifacts.use=true` Use the new version number when prompted.
+4. `mvn release:perform -Dh3.github.artifacts.use=true`
 5. If this looks good, close and release the build in [Sonatype Nexus Manager](https://oss.sonatype.org/).
 6. Update `CHANGELOG.md` to have an Unreleased section, and commit. The release is now done and development can resume from this point.
 
 ## Troubleshooting
 
+### Dependencies for `pull-from-github.sh`
+
+* You should install the [Github CLI](https://cli.github.com) and authenticate with it first. You may need to use a personal access token (classic) with workflows scope.
+* `jq`
+
 ### gpg: signing failed: Inappropriate ioctl for device
 
 Per [StackOverflow](https://stackoverflow.com/questions/57591432/gpg-signing-failed-inappropriate-ioctl-for-device-on-macos-with-maven), run the following before `mvn release:perform`:
diff --git a/pom.xml b/pom.xml
@@ -18,7 +18,7 @@
     <groupId>com.uber</groupId>
     <artifactId>h3</artifactId>
     <packaging>jar</packaging>
-    <version>4.0.3-SNAPSHOT</version>
+    <version>4.1.0</version>
     <name>h3</name>
     <url>https://github.com/uber/h3-java</url>
     <description>Java bindings for H3, a hierarchical hexagonal geospatial indexing system.</description>
@@ -40,7 +40,7 @@
         <connection>scm:git:git://github.com/uber/h3-java.git</connection>
         <developerConnection>scm:git:ssh://git@github.com/uber/h3-java.git</developerConnection>
         <url>http://github.com/uber/h3-java/tree/master</url>
-      <tag>HEAD</tag>
+      <tag>v4.1.0</tag>
   </scm>
 
     <developers>
@@ -72,6 +72,8 @@
         <h3.use.docker>true</h3.use.docker>
         <h3.system.prune>false</h3.system.prune>
         <h3.dockcross.tag>20210624-de7b1b0</h3.dockcross.tag>
+        <h3.github.artifacts.use>false</h3.github.artifacts.use>
+        <h3.github.artifacts.by_run />
         <!-- Used for passing additional arguments to surefire when using JaCoCo -->
         <h3.additional.argLine />
 
@@ -112,7 +114,7 @@
                     <plugin>
                         <groupId>org.codehaus.mojo</groupId>
                         <artifactId>exec-maven-plugin</artifactId>
-                        <version>1.1.1</version>
+                        <version>3.1.0</version>
                         <executions>
                             <execution>
                                 <!-- Override when building on Windows to use PowerShell & MSVC -->
@@ -127,7 +129,7 @@
                                         <argument>-ExecutionPolicy</argument>
                                         <argument>Bypass</argument>
                                         <argument>-File</argument>
-                                        <argument>${basedir}/src/main/c/h3-java/build-h3-windows.ps1</argument>
+                                        <argument>${project.basedir}/src/main/c/h3-java/build-h3-windows.ps1</argument>
                                         <argument>${h3.git.remote}</argument>
                                         <argument>${h3.git.reference}</argument>
                                     </arguments>
@@ -203,7 +205,7 @@
                         </goals>
                         <configuration>
                             <files>
-                                <file>${basedir}/h3version.properties</file>
+                                <file>${project.basedir}/h3version.properties</file>
                             </files>
                         </configuration>
                     </execution>
@@ -223,7 +225,7 @@
 
                         <!-- Generate the native header file -->
                         <arg>-h</arg>
-                        <arg>${basedir}/src/main/c/h3-java/src</arg>
+                        <arg>${project.basedir}/src/main/c/h3-java/src</arg>
                     </compilerArgs>
                 </configuration>
             </plugin>
@@ -250,7 +252,7 @@
             <plugin>
                 <groupId>org.codehaus.mojo</groupId>
                 <artifactId>exec-maven-plugin</artifactId>
-                <version>1.1.1</version>
+                <version>3.1.0</version>
                 <executions>
                     <execution>
                         <id>build-h3-c</id>
@@ -261,13 +263,15 @@
                             <goal>exec</goal>
                         </goals>
                         <configuration>
-                            <executable>${basedir}/src/main/c/h3-java/build-h3.sh</executable>
+                            <executable>${project.basedir}/src/main/c/h3-java/build-h3.sh</executable>
                             <arguments>
                                 <argument>${h3.git.remote}</argument>
                                 <argument>${h3.git.reference}</argument>
                                 <argument>${h3.use.docker}</argument>
                                 <argument>${h3.system.prune}</argument>
                                 <argument>${h3.dockcross.tag}</argument>
+                                <argument>${h3.github.artifacts.use}</argument>
+                                <argument>${h3.github.artifacts.by_run}</argument>
                             </arguments>
                         </configuration>
                     </execution>
@@ -288,7 +292,7 @@
                             <outputDirectory>${project.build.outputDirectory}</outputDirectory>
                             <resources>
                                 <resource>
-                                    <directory>${basedir}/src/main/resources</directory>
+                                    <directory>${project.basedir}/src/main/resources</directory>
                                     <filtering>false</filtering>
                                 </resource>
                             </resources>
diff --git a/src/main/c/h3-java/build-h3.sh b/src/main/c/h3-java/build-h3.sh
@@ -15,16 +15,21 @@
 # limitations under the License.
 #
 
-# Arguments: [git-remote] [git-ref] [use-docker] [remove-images]
-# git-remote    - The git remote to pull from. An existing cloned repository
-#                 will not be deleted if a new remote is specified.
-# git-ref       - Specific git ref of H3 to build.
-# use-docker    - "true" to perform cross compilation via Docker, "false" to
-#                 skip that step.
-# system-prune  - If use-docker is true and this argument is true, Docker
-#                 system prune will be run after each step
-#                 (i.e. for disk space constrained environments like CI)
-# dockcross-tag - Tag name for dockcross
+# Arguments: [git-remote] [git-ref] [use-docker] [remove-images] [github-artifacts] [github-artifacts-ref]
+# git-remote       - The git remote to pull from. An existing cloned repository
+#                    will not be deleted if a new remote is specified.
+# git-ref          - Specific git ref of H3 to build.
+# use-docker       - "true" to perform cross compilation via Docker, "false" to
+#                    skip that step.
+# system-prune     - If use-docker is true and this argument is true, Docker
+#                    system prune will be run after each step
+#                    (i.e. for disk space constrained environments like CI)
+# dockcross-tag    - Tag name for dockcross
+# github-artifacts - When set, all build artifacts are retrieved from Github
+#                    Actions artifacts rather than built locally (overrides
+#                    all other settings.)
+# github-artifacts-run - When set, assume the artifacts have already been
+#                        downloaded.
 #
 # This script downloads H3, builds H3 and the H3-Java native library, and
 # cross compiles via Docker.
@@ -39,6 +44,13 @@ GIT_REVISION=$2
 USE_DOCKER=$3
 SYSTEM_PRUNE=$4
 DOCKCROSS_TAG=$5
+GITHUB_ARTIFACTS=$6
+GITHUB_ARTIFACTS_RUN=$7
+
+if $GITHUB_ARTIFACTS; then
+    src/main/c/h3-java/pull-from-github.sh "$GITHUB_ARTIFACTS_RUN"
+    # The build of the library below is still needed for the binding-functions file
+fi
 
 echo Downloading H3 from "$GIT_REMOTE"
 
@@ -90,6 +102,11 @@ cp h3-java-build/build/binding-functions .
 
 popd # target
 
+if $GITHUB_ARTIFACTS; then
+    # Nothing more is needed than the binding-functions to stop now.
+    exit 0
+fi
+
 # Copy the built artifact for this platform.
 case "$(uname -sm)" in
     "Linux x86_64")  LIBRARY_DIR=linux-x64 ;;
diff --git a/src/main/c/h3-java/pull-from-github.sh b/src/main/c/h3-java/pull-from-github.sh
@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+#
+# Copyright 2023 Uber Technologies, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Retrieves built artifacts from Github. The purpose of this script is
+# to enable deployment from any system, regardless of whether it can
+# run the particular cross compiling steps needed to build native shared
+# objects. Some of the native shared objects require particular Docker
+# setups, or indeed running on Mac OS, which is difficult to maintain.
+#
+# You must install the Github CLI and authenticate with a personal access
+# token (classic) with workflows scope to use this. https://cli.github.com/
+# This script also requires `jq`.
+#
+# This script expects to be run from the project's base directory (where
+# pom.xml is) as part of the Maven build process.
+
+set -eox pipefail
+
+GITHUB_ARTIFACTS_RUN=$1
+
+EXTRACT_TO=src/main/resources
+
+if [ -z "$GITHUB_ARTIFACTS_RUN" ]; then
+    mkdir -p target
+    pushd target
+
+    ARTIFACTS_LIST=$(gh api \
+    -H "Accept: application/vnd.github+json" \
+    /repos/{owner}/{repo}/actions/artifacts)
+
+    echo "downloading artifacts for run $GITHUB_ARTIFACTS_RUN"
+    TO_DOWNLOAD=$(echo "$ARTIFACTS_LIST" \
+        | jq ".artifacts[] | select(.workflow_run.id == \"$GITHUB_ARTIFACTS_RUN\")")
+
+    echo $TO_DOWNLOAD | jq -c '.' | while read artifactline; do
+        ARTIFACT_NAME=$(echo $artifactline | jq -r .name)
+        ARTIFACT_ID=$(echo $artifactline | jq .id)
+        echo "Downloading $ARTIFACT_NAME: $ARTIFACT_ID"
+        gh api "/repos/{owner}/{repo}/actions/artifacts/$ARTIFACT_ID/zip" > "$ARTIFACT_NAME.zip"
+        unzip -o "$ARTIFACT_NAME.zip" -d "../$EXTRACT_TO"
+    done
+
+    popd
+fi
+
+echo Checking that expected images are present:
+
+for image in android-arm android-arm64 linux-arm64 linux-armv5 linux-armv7 linux-s390x \
+    linux-ppc64le linux-x64 linux-x86; do
+    if [ -f "$EXTRACT_TO/$image/libh3-java.so" ]; then
+        echo "$image" exists
+    else
+        echo "$image" missing!
+        exit 1
+    fi
+done
+
+for image in darwin-x64 darwin-arm64; do
+    if [ -f "$EXTRACT_TO/$image/libh3-java.dylib" ]; then
+        echo "$image" exists
+    else
+        echo "$image" missing!
+        exit 1
+    fi
+done
+
+for image in windows-x64 windows-x86; do
+    if [ -f "$EXTRACT_TO/$image/libh3-java.dll" ]; then
+        echo "$image" exists
+    else
+        echo "$image" missing!
+        exit 1
+    fi
+done
