Merge #833

833: Feature flag for custom Godot version r=Bromeon a=Bromeon

Enables the use of custom Godot builds in a straightforward way.
This includes older versions such as Godot 3.2, for which `api.json` is no longer shipped along.

Previous process (see also [book](https://godot-rust.github.io/book/advanced-guides/custom-godot.html)):
* create a local copy of godot-rust
* Run `godot --gdnative-generate-json-api`
* replace `api.json` inside `gdnative-binding`

New process:
* enable feature `custom-godot`
* make sure `godot` executable is in path **OR** set `GODOT_BIN` env var
* done

This supersedes the previous `bindings` feature, which is now removed.

I also added CI support for previous Godot version 3.3.1, so far only for integration tests. Originally I wanted to use Godot 3.2 (the oldest supported version), however godotengine/godot#36582 prevented headless versions from generating `api.json` reliably without crashing. This has been fixed for versions >= 3.3.1.

There is still testing to be done; feedback is always appreciated! 🙂 

Closes #640. Thanks for the great ideas in that issue.

Co-authored-by: Jan Haller <[email protected]>

Author: bors[bot]

.github/composite/godot/action.yml

@@ -0,0 +1,82 @@
+name: godot
+description: "Run Godot integration tests"
+
+inputs:
+  godot_ver:
+    required: true
+    description: "Godot version (e.g. '3.2')"
+
+  rust_toolchain:
+    required: false
+    default: 'stable'
+    description: "Rust toolchain specifier (e.g. 'nightly')"
+
+  rust_extra_args:
+    required: false
+    default: ''
+    description: "Extra command line arguments for 'cargo build', e.g. features"
+
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@v2
+    - name: Install Rust
+      uses: ./.github/composite/rust
+      with:
+        rust: ${{ inputs.rust_toolchain }}
+    - name: "Check cache for installed Godot version"
+      id: "cache-godot"
+      uses: actions/cache@v2
+      with:
+        path: ${{ runner.temp }}/godot_bin
+        key: godot-${{ runner.os }}-v${{ inputs.godot_ver }}
+    - name: "Install Godot"
+      if: steps.cache-godot.outputs.cache-hit != 'true'
+      run: |
+        wget --no-verbose "https://downloads.tuxfamily.org/godotengine/${{ inputs.godot_ver }}/Godot_v${{ inputs.godot_ver }}-stable_linux_headless.64.zip" -O /tmp/godot.zip
+        unzip -o /tmp/godot.zip -d ${{ runner.temp }}/godot_bin
+      shell: bash
+    - name: "Set environment variable"
+      run: |
+        echo "GODOT_BIN=${{ runner.temp }}/godot_bin/Godot_v${{ inputs.godot_ver }}-stable_linux_headless.64" >> $GITHUB_ENV
+      shell: bash
+    - name: "Build godot-rust"
+      run: |
+        echo "File size of api.json -- before:"
+        stat -c %s gdnative-bindings/api.json
+        
+        #echo "CRC32 of api.json: "
+        #crc32 gdnative-bindings/api.json
+        cd test
+        cargo build ${{ inputs.rust_extra_args }}
+        
+        cd ..
+        echo "File size of api.json -- after:"
+        stat -c %s gdnative-bindings/api.json
+      shell: bash
+    - name: "Run Godot integration tests"
+      run: |
+        cd test;
+        mkdir -p ./project/lib;
+        cp ../target/debug/libgdnative_test.so ./project/lib/;
+        ${GODOT_BIN} --path ./project/ > >(tee "${{ runner.temp }}/stdout.log");
+        if grep -q "Leaked instance" "${{ runner.temp }}/stdout.log"; then
+          exit 1;
+        fi;
+        ${GODOT_BIN} -e --path ./project/ --run-editor-tests > >(tee "${{ runner.temp }}/stdout.log");
+        if grep -q "Leaked instance" "${{ runner.temp }}/stdout.log"; then
+          exit 1;
+        fi;
+        cargo build --features type-tag-fallback;
+        mkdir -p ./project/lib;
+        cp ../target/debug/libgdnative_test.so ./project/lib/;
+        ${GODOT_BIN} --path ./project/ > >(tee "${{ runner.temp }}/stdout.log");
+        if grep -q "Leaked instance" "${{ runner.temp }}/stdout.log"; then
+          exit 1;
+        fi;
+        ${GODOT_BIN} -e --path ./project/ --run-editor-tests > >(tee "${{ runner.temp }}/stdout.log");
+        if grep -q "Leaked instance" "${{ runner.temp }}/stdout.log"; then
+          exit 1;
+        fi;
+      shell: bash

.github/workflows/doc.yml

@@ -14,13 +14,14 @@ on:
 
 
 env:
-  GDNATIVE_LIB_RS_PREFIX: |-
+  GDRUST_LIB_RS_PREFIX: |-
       //! _**Note:** This documentation refers to the [latest GitHub version](https://github.com/godot-rust/godot-rust) and is subject to change._<br>
       //! _For stable releases, visit [docs.rs/gdnative](https://docs.rs/gdnative)._
       //! <br><br>
       //!
-  GDNATIVE_DOC_REPO: [email protected]:godot-rust/docs.git
-  GDNATIVE_DOC_BRANCH: gh-pages
+  GDRUST_DOC_REPO: [email protected]:godot-rust/docs.git
+  GDRUST_DOC_BRANCH: gh-pages
+  GDRUST_FEATURES: "async,serde"
 
 
 # In the very unlikely cases where two PRs are merged, and the first 'doc' job is still running when the 2nd 'full-ci' starts,
@@ -46,13 +47,13 @@ jobs:
       - name: "Pre-process input"
         run: |
           mv ${GITHUB_WORKSPACE}/gdnative/src/lib.rs tmp_lib.rs
-          (echo "${GDNATIVE_LIB_RS_PREFIX}"; cat tmp_lib.rs) > ${GITHUB_WORKSPACE}/gdnative/src/lib.rs
+          (echo "${GDRUST_LIB_RS_PREFIX}"; cat tmp_lib.rs) > ${GITHUB_WORKSPACE}/gdnative/src/lib.rs
 
       - name: "Generate documentation"
         uses: actions-rs/cargo@v1
         with:
           command: doc
-          args: -p gdnative --lib --no-deps --all-features
+          args: -p gdnative --lib --no-deps --features ${GDRUST_FEATURES}
 
       # For email address, see https://github.community/t/github-actions-bot-email-address/17204
       # As search-index.js changes every time, even if source hasn't changed, this will not need 'git commit --allow-empty'
@@ -65,25 +66,25 @@ jobs:
           git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
 
           mkdir doc && cd doc
-          git clone --single-branch --branch ${GDNATIVE_DOC_BRANCH} --no-checkout ${GDNATIVE_DOC_REPO} . \
-            || (git init -b ${GDNATIVE_DOC_BRANCH} && git remote add origin ${GDNATIVE_DOC_REPO})
+          git clone --single-branch --branch ${GDRUST_DOC_BRANCH} --no-checkout ${GDRUST_DOC_REPO} . \
+            || (git init -b ${GDRUST_DOC_BRANCH} && git remote add origin ${GDRUST_DOC_REPO})
 
           mv ${GITHUB_WORKSPACE}/target/doc/* .
           mv ${GITHUB_WORKSPACE}/.github/workflows/doc/* .
 
-          GDNATIVE_VERSION=$(grep -Po '^version = "\K[^"]*' ${GITHUB_WORKSPACE}/gdnative/Cargo.toml)
-          GDNATIVE_SHORT_SHA=$(git rev-parse --short "${GITHUB_SHA}")
+          GDRUST_VERSION=$(grep -Po '^version = "\K[^"]*' ${GITHUB_WORKSPACE}/gdnative/Cargo.toml)
+          GDRUST_SHORT_SHA=$(git rev-parse --short "${GITHUB_SHA}")
 
-          find gdnative -name .html -o -type f -print0 | xargs -0 sed -i 's/'"${GDNATIVE_VERSION}"'/master/g'
+          find gdnative -name .html -o -type f -print0 | xargs -0 sed -i 's/'"${GDRUST_VERSION}"'/master/g'
 
           git add --all
-          git commit -m "Sync doc from ${GDNATIVE_SHORT_SHA}
+          git commit -m "Sync doc from ${GDRUST_SHORT_SHA}
 
           Revision in godot-rust: ${GITHUB_SHA}"
 
       - name: "Upload"
         working-directory: doc
-        run: git push origin ${GDNATIVE_DOC_BRANCH}
+        run: git push origin ${GDRUST_DOC_BRANCH}
 
       - name: "Cleanup"
         run: shred -u ~/.ssh/id_rsa

.github/workflows/full-ci.yml

@@ -32,8 +32,7 @@ env:
 
   # Local variables
   # Note: using variables is limited at the moment, see https://github.com/actions/runner/issues/480
-  GODOT_VER: "3.4"
-  GODOT_REL: stable
+  GDRUST_FEATURES: "gdnative/async,gdnative/serde"
 
 on:
   push:
@@ -50,11 +49,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Install Rust
+      - name: "Install Rust"
         uses: ./.github/composite/rust
         with:
           components: rustfmt
-      - name: Check rustfmt
+      - name: "Check rustfmt"
         run: cargo fmt --all -- --check;
 
   clippy:
@@ -71,13 +70,13 @@ jobs:
             postfix: ' (nightly)'
     steps:
       - uses: actions/checkout@v2
-      - name: Install Rust
+      - name: "Install Rust"
         uses: ./.github/composite/rust
         with:
           rust: ${{ matrix.rust.toolchain }}
           components: clippy
-      - name: Check clippy
-        run: cargo clippy --workspace --all-features -- -D clippy::style -D clippy::complexity -D clippy::perf -D clippy::dbg_macro -D clippy::todo -D clippy::unimplemented;
+      - name: "Check clippy"
+        run: cargo clippy --workspace --features ${GDRUST_FEATURES} -- -D clippy::style -D clippy::complexity -D clippy::perf -D clippy::dbg_macro -D clippy::todo -D clippy::unimplemented;
 
   test:
     name: test-${{ matrix.os.name }}${{ matrix.rust.postfix }}
@@ -106,22 +105,22 @@ jobs:
           - rust: { toolchain: 'nightly' }
             testflags: '-- --skip ui_tests'
           - os: { id: ubuntu-latest, name: linux }
-            rust: { toolchain: '1.48', postfix: ' (msrv 1.48)' }
+            rust: { toolchain: '1.51', postfix: ' (msrv 1.51)' }
             testflags: '-- --skip ui_tests'
     runs-on: ${{ matrix.os.id }}
     steps:
       - uses: actions/checkout@v2
-      - name: Install Rust
+      - name: "Install Rust"
         uses: ./.github/composite/rust
         with:
           rust: ${{ matrix.rust.toolchain }}
-      - name: Install LLVM
+      - name: "Install LLVM"
         uses: ./.github/composite/llvm
         if: ${{ matrix.os.id == 'windows-latest' }}
-      - name: Compile tests
-        run: cargo test --workspace --all-features --no-run;
-      - name: Test
-        run: cargo test --workspace --all-features ${{ matrix.testflags }};
+      - name: "Compile tests"
+        run: cargo test --workspace --features ${GDRUST_FEATURES} --no-run;
+      - name: "Test"
+        run: cargo test --workspace --features ${GDRUST_FEATURES} ${{ matrix.testflags }};
 
   build-release:
     name: build-release-${{ matrix.os.name }}
@@ -139,14 +138,14 @@ jobs:
     runs-on: ${{ matrix.os.id }}
     steps:
       - uses: actions/checkout@v2
-      - name: Install Rust
+      - name: "Install Rust"
         uses: ./.github/composite/rust
         with:
           rust: stable
-      - name: Install LLVM
+      - name: "Install LLVM"
         uses: ./.github/composite/llvm
         if: ${{ matrix.os.id == 'windows-latest' }}
-      - name: Release build (check only)
+      - name: "Release build (check only)"
         run: cargo check --release;
 
   build-ios:
@@ -158,18 +157,18 @@ jobs:
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Install Rust
+      - name: "Install Rust"
         uses: ./.github/composite/rust
         #with:
         #  rust: ${{ matrix.rust.toolchain }}
-      - name: Install cargo-dinghy
+      - name: "Install cargo-dinghy"
         run: |
           rustup target add x86_64-apple-ios;
           curl -L https://github.com/sonos/dinghy/releases/download/0.4.62/cargo-dinghy-macos-0.4.62.tgz -o cargo-dinghy-macos.tar.gz;
           tar -zxvf cargo-dinghy-macos.tar.gz;
           mkdir -p $HOME/.cargo/bin;
           cp cargo-dinghy-0.4.62/cargo-dinghy $HOME/.cargo/bin;
-      - name: Cross-compile to iOS
+      - name: "Cross-compile to iOS"
         run: |
           RUNTIME_ID=$(xcrun simctl list runtimes | grep iOS | cut -d ' ' -f 7 | tail -1);
           export SIM_ID=$(xcrun simctl create My-iphone11 com.apple.CoreSimulator.SimDeviceType.iPhone-11 $RUNTIME_ID);
@@ -190,11 +189,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Install Rust
+      - name: "Install Rust"
         uses: ./.github/composite/rust
         #with:
         #  rust: ${{ matrix.rust.toolchain }}
-      - name: Install Java + NDK
+      - name: "Install Java + NDK"
         run: |
           export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64;
           export ANDROID_SDK_ROOT=/opt/ndk/android-ndk-r21d;
@@ -227,65 +226,42 @@ jobs:
           cargo build --target armv7-linux-androideabi --release;
 
   integration-test-godot:
-    name: itest-godot${{ matrix.rust.postfix }}
+    name: itest-godot-${{ matrix.godot }}${{ matrix.postfix }}
     needs: rustfmt
     continue-on-error: ${{ matrix.rust.toolchain == 'nightly' }}
     strategy:
+      fail-fast: false # cancel all jobs as soon as one fails?
       matrix:
-        rust:
-          - toolchain: stable
+        include:
+          # Latest Godot with different Rust versions
+          - rust: stable
+            godot: "3.4.1"
             postfix: ''
-          - toolchain: nightly
+          - rust: nightly
+            godot: "3.4.1"
             postfix: ' (nightly)'
-          - toolchain: '1.48'
-            postfix: ' (msrv 1.48)'
+          - rust: '1.51'
+            godot: "3.4.1"
+            postfix: ' (msrv 1.51)'
+
+          # Test with older engine version
+          # Note: headless versions of Godot <= 3.3 may crash with a bug, see feature description in lib.rs
+          - rust: stable
+            godot: "3.3.1"
+            postfix: ''
+            build_args: '--features custom-godot'
+
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Install Rust
-        uses: ./.github/composite/rust
+      - name: "Run Godot integration test"
+        uses: ./.github/composite/godot
         with:
-          rust: ${{ matrix.rust.toolchain }}
-      - name: Check cache for installed Godot version
-        id: cache-godot
-        uses: actions/cache@v2
-        with:
-          path: ${{ runner.temp }}/godot_bin
-          key: godot-${{ runner.os }}-v${{ env.GODOT_VER }}-${{ env.GODOT_REL }}
-      - name: Install Godot
-        if: steps.cache-godot.outputs.cache-hit != 'true'
-        run: |
-          wget "https://downloads.tuxfamily.org/godotengine/$GODOT_VER/Godot_v${GODOT_VER}-${GODOT_REL}_linux_headless.64.zip" -O /tmp/godot.zip
-          unzip /tmp/godot.zip -d ${{ runner.temp }}/godot_bin
-      - name: Build godot-rust
-        run: |
-          cd test;
-          cargo build;
-      - name: Run Godot integration tests
-        run: |
-          cd test;
-          mkdir -p ./project/lib;
-          cp ../target/debug/libgdnative_test.so ./project/lib/;
-          "${{ runner.temp }}/godot_bin/Godot_v${GODOT_VER}-${GODOT_REL}_linux_headless.64" --path ./project/ > >(tee "${{ runner.temp }}/stdout.log");
-          if grep -q "Leaked instance" "${{ runner.temp }}/stdout.log"; then
-            exit 1;
-          fi;
-          "${{ runner.temp }}/godot_bin/Godot_v${GODOT_VER}-${GODOT_REL}_linux_headless.64" -e --path ./project/ --run-editor-tests > >(tee "${{ runner.temp }}/stdout.log");
-          if grep -q "Leaked instance" "${{ runner.temp }}/stdout.log"; then
-            exit 1;
-          fi;
-          cargo build --features=type_tag_fallback;
-          mkdir -p ./project/lib;
-          cp ../target/debug/libgdnative_test.so ./project/lib/;
-          "${{ runner.temp }}/godot_bin/Godot_v${GODOT_VER}-${GODOT_REL}_linux_headless.64" --path ./project/ > >(tee "${{ runner.temp }}/stdout.log");
-          if grep -q "Leaked instance" "${{ runner.temp }}/stdout.log"; then
-            exit 1;
-          fi;
-          "${{ runner.temp }}/godot_bin/Godot_v${GODOT_VER}-${GODOT_REL}_linux_headless.64" -e --path ./project/ --run-editor-tests > >(tee "${{ runner.temp }}/stdout.log");
-          if grep -q "Leaked instance" "${{ runner.temp }}/stdout.log"; then
-            exit 1;
-          fi;
-          
+          rust_toolchain: ${{ matrix.rust }}
+          rust_extra_args: ${{ matrix.build_args }}
+          godot_ver: ${{ matrix.godot }}
+
+
   # This job doesn't actually test anything, but is used to tell bors that the build completed,
   # as there is no practical way to detect when a workflow is successful, listening to webhooks only.
   # The ID (not name) of this job is the one referenced in bors.toml.
@@ -303,5 +279,5 @@ jobs:
       - build-android
     runs-on: ubuntu-latest
     steps:
-      - name: Mark the job as a success
+      - name: "Mark the job as a success"
         run: exit 0