Feat: support serialization for erlang terms (#2)

* Feat: ADR for hybrid storage

* Chore: ignore the priv folder

* Chore: remove target folder from GIT

* Chore: remove priv/native folder from GIT

* Feat: implement support for arbitrary Erlang Term values

* Chore: remove checksum from GIT history

* Chore: add couple bash scripts for clean compile / tests

* Chore: remove old stuff from readmes

* Chore: group release binaries by target platform

* Chore: pull latest release from main - trigger

* Chore: try building release on PR branch

* Chore: try new release pipeline to support MUSL

* Feat: try rustler configs by example of https://github.com/leandrocp/mdex

* chore

* Feat: add .cargo/config.toml

* Feat: add support for rust in single pipeline

* Chore: adjust formatting

* Chore: fix rust formatting

* Chore: fix clippy warning

* Chore: add more checks to bin/test.sh

* Chore: add windows + apple x86

* Feat: add script to generate checksum file + ignore it for GIT

Author: mindreframer

.github/workflows/ci.yml

@@ -5,6 +5,7 @@ on:
   push:
     branches:
       - master
+      - feat/support_for_erlang_terms
 
 jobs:
   mix_test:
@@ -51,4 +52,10 @@ jobs:
       - run: mix compile --warnings-as-errors
         if: ${{ matrix.lint }}
 
-      - run: mix test
+      - run: mix test
+
+      - name: run rustfmt
+        run: cargo fmt --manifest-path=native/trie_hard_native/Cargo.toml --all -- --check
+
+      - name: run clippy
+        run: cargo clippy --manifest-path=native/trie_hard_native/Cargo.toml -- -Dwarnings

.github/workflows/release.yml

@@ -1,108 +1,111 @@
-name: Build precompiled NIFs
+# https://github.com/elixir-explorer/explorer/blob/main/.github/workflows/release.yml
+# https://github.com/Takashiidobe/rust-build-binary-github-actions/blob/master/.github/workflows/release.yml
 
-permissions:
-  id-token: write
-  attestations: write
-  contents: write
+name: Precomp NIFs
 
 on:
   push:
     branches:
-      - main
       - master
+      - feat/support_for_erlang_terms
     paths:
-      # Just run on main branch if "native" path changed.
       - "native/**"
-      # Also run if this file changes.
       - ".github/workflows/release.yml"
     tags:
-      # Tags will always run.
       - "*"
   pull_request:
     paths:
-      # In PRs we only run if this file changes.
       - ".github/workflows/release.yml"
   workflow_dispatch:
 
 jobs:
   build_release:
-    name: NIF ${{ matrix.nif }} - ${{ matrix.job.target }} (${{ matrix.job.os }})
+    name: NIF ${{ matrix.nif }} - ${{ matrix.job.target }} (${{ matrix.job.os }} | ${{ matrix.job.variant || 'default' }})
     runs-on: ${{ matrix.job.os }}
+
+    permissions:
+      contents: write
+      id-token: write
+      attestations: write
+
     strategy:
       fail-fast: false
       matrix:
         nif: ["2.15", "2.16"]
         job:
-          - { target: aarch64-unknown-linux-gnu   , os: ubuntu-22.04 , use-cross: true }
-          - { target: aarch64-unknown-linux-musl  , os: ubuntu-22.04 , use-cross: true }
-          - { target: x86_64-unknown-linux-gnu    , os: ubuntu-22.04  }
-          - { target: x86_64-unknown-linux-musl   , os: ubuntu-22.04 , use-cross: true }
-          - { target: aarch64-apple-darwin        , os: macos-13      }
-          - { target: x86_64-apple-darwin         , os: macos-13      }
-          - { target: x86_64-pc-windows-msvc      , os: windows-2022  }
+          - { target: aarch64-apple-darwin,         os: macos-13 }
+          - { target: aarch64-unknown-linux-gnu,    os: ubuntu-22.04, use-cross: true }
+          - { target: aarch64-unknown-linux-musl,   os: ubuntu-22.04, use-cross: true }
+          # - { target: arm-unknown-linux-gnueabihf,  os: ubuntu-22.04, use-cross: true }
+          # - { target: riscv64gc-unknown-linux-gnu,  os: ubuntu-22.04, use-cross: true }
+          - { target: x86_64-apple-darwin,          os: macos-13 }
+          - { target: x86_64-pc-windows-gnu,        os: windows-2022,                  rustflags: "-C target-feature=+fxsr,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma" }
+          - { target: x86_64-pc-windows-gnu,        os: windows-2022,                  variant: "legacy_cpu" }
+          # - { target: x86_64-pc-windows-msvc,       os: windows-2022,                  rustflags: "-C target-feature=+fxsr,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma" }
+          # - { target: x86_64-pc-windows-msvc,       os: windows-2022,                  variant: "legacy_cpu" }
+          # - { target: x86_64-unknown-freebsd,       os: ubuntu-22.04, use-cross: true, rustflags: "-C target-feature=+fxsr,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma" }
+          # - { target: x86_64-unknown-freebsd,       os: ubuntu-22.04, use-cross: true, variant: "legacy_cpu" }
+          - { target: x86_64-unknown-linux-gnu,     os: ubuntu-22.04,                  rustflags: "-C target-feature=+fxsr,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma" }
+          - { target: x86_64-unknown-linux-gnu,     os: ubuntu-22.04,                  variant: "legacy_cpu" }
+          - { target: x86_64-unknown-linux-musl,    os: ubuntu-22.04, use-cross: true }
 
     steps:
-    - name: Checkout source code
-      uses: actions/checkout@v4
+      - name: Checkout source code
+        uses: actions/checkout@v5
 
-    - name: Extract project version
-      shell: bash
-      run: |
-        # Get the version from mix.exs
-        VERSION=$(grep '@version' mix.exs | head -n 1 | sed 's/.*@version "\(.*\)".*/\1/')
-        echo "PROJECT_VERSION=$VERSION" >> $GITHUB_ENV
+      - name: Extract crate information
+        shell: bash
+        run: |
+          # Get the project version from mix.exs
+          echo "PROJECT_VERSION=$(sed -n 's/^  @version "\(.*\)"/\1/p' mix.exs | head -n1)" >> $GITHUB_ENV
 
-    - name: Install Elixir and Erlang
-      uses: erlef/setup-beam@v1
-      with:
-        version-type: strict
-        otp-version: 26.2
-        elixir-version: 1.18.4
+      - name: Maybe add Rust flags for compilation
+        shell: bash
+        run: |
+          echo "RUSTFLAGS=${{ matrix.job.rustflags }}" >> $GITHUB_ENV
+        if: ${{ matrix.job.rustflags }}
 
-    - name: Install Rust toolchain
-      uses: dtolnay/rust-toolchain@stable
-      with:
-        target: ${{ matrix.job.target }}
+      - name: Add target
+        shell: bash
+        run: |
+          rustup target add ${{ matrix.job.target }}
 
-    - name: Cache Rust dependencies
-      uses: Swatinem/rust-cache@v2
-      with:
-        workspaces: native/trie_hard_native
-        key: ${{ matrix.job.target }}-${{ matrix.nif }}
+      - uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: v0-precomp
+          shared-key: ${{ matrix.job.target }}-${{ matrix.nif }}
+          workspaces: |
+            native/trie_hard_native
 
-    - name: Cache Mix dependencies
-      uses: actions/cache@v4
-      with:
-        path: |
-          deps
-          _build
-        key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-mix-
+      - name: Build the project
+        id: build-crate
+        uses: philss/rustler-precompiled-action@v1.1.4
+        with:
+          project-name: trie_hard_native
+          project-version: ${{ env.PROJECT_VERSION }}
+          target: ${{ matrix.job.target }}
+          nif-version: ${{ matrix.nif }}
+          use-cross: ${{ matrix.job.use-cross }}
+          cross-version: ${{ matrix.job.cross-version || 'from-source' }}
+          project-dir: "native/trie_hard_native"
+          cargo-args: ${{ matrix.job.cargo-args }}
+          variant: ${{ matrix.job.variant }}
 
-    - name: Install dependencies
-      run: mix deps.get
 
-    - name: Build the project
-      id: build-crate
-      uses: philss/rustler-precompiled-action@v1.1.4
-      with:
-        project-name: trie_hard_native
-        project-version: ${{ env.PROJECT_VERSION }}
-        target: ${{ matrix.job.target }}
-        nif-version: ${{ matrix.nif }}
-        use-cross: ${{ matrix.job.use-cross }}
-        project-dir: "native/trie_hard_native"
+      - name: Artifact attestation
+        uses: actions/attest-build-provenance@v3
+        with:
+          subject-path: ${{ steps.build-crate.outputs.file-path }}
 
-    - name: Artifact upload
-      uses: actions/upload-artifact@v4
-      with:
-        name: ${{ steps.build-crate.outputs.file-name }}
-        path: ${{ steps.build-crate.outputs.file-path }}
+      - name: Artifact upload
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.build-crate.outputs.file-name }}
+          path: ${{ steps.build-crate.outputs.file-path }}
 
-    - name: Publish archives and packages
-      uses: softprops/action-gh-release@v2
-      with:
-        files: |
-          ${{ steps.build-crate.outputs.file-path }}
-      if: startsWith(github.ref, 'refs/tags/')
+      - name: Publish archives and packages
+        uses: softprops/action-gh-release@v2
+        with:
+          files: |
+            ${{ steps.build-crate.outputs.file-path }}
+        if: startsWith(github.ref, 'refs/tags/')

.github/workflows/rust-ci.yml

@@ -25,3 +25,7 @@ trie_again-*.tar
 # Rust binary artifacts
 /target/
 native/**/target/
+priv/
+
+# The checksum files for precompiled NIFs
+checksum-*.exs