Merge tag 'v0.17.0' into jdk-21+24-mmtk-merge

Author: caizixian

.github/scripts/ci-checkout.sh

@@ -3,8 +3,8 @@ set -ex
 
 . $(dirname "$0")/common.sh
 
-OPENJDK_URL=`cargo read-manifest --manifest-path=$BINDING_PATH/mmtk/Cargo.toml | python -c 'import json,sys; print(json.load(sys.stdin)["metadata"]["openjdk"]["openjdk_repo"])'`
-OPENJDK_VERSION=`cargo read-manifest --manifest-path=$BINDING_PATH/mmtk/Cargo.toml | python -c 'import json,sys; print(json.load(sys.stdin)["metadata"]["openjdk"]["openjdk_version"])'`
+OPENJDK_URL=`cargo read-manifest --manifest-path=$BINDING_PATH/mmtk/Cargo.toml | python3 -c 'import json,sys; print(json.load(sys.stdin)["metadata"]["openjdk"]["openjdk_repo"])'`
+OPENJDK_VERSION=`cargo read-manifest --manifest-path=$BINDING_PATH/mmtk/Cargo.toml | python3 -c 'import json,sys; print(json.load(sys.stdin)["metadata"]["openjdk"]["openjdk_version"])'`
 
 rm -rf $OPENJDK_PATH
 git clone $OPENJDK_URL $OPENJDK_PATH

.github/scripts/ci-test-only-normal.sh

@@ -23,6 +23,9 @@ build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHea
 build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb
 build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar eclipse
 
+# Test heap resizing
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms20M -Xmx100M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
+
 # --- Immix ---
 export MMTK_PLAN=Immix
 
@@ -41,6 +44,9 @@ build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHea
 build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb
 build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar eclipse
 
+# Test heap resizing
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms20M -Xmx100M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
+
 # --- Immix ---
 export MMTK_PLAN=GenImmix
 
@@ -59,6 +65,9 @@ build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHea
 build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb
 build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar eclipse
 
+# Test heap resizing
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms20M -Xmx100M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
+
 # --- GenCopy ---
 export MMTK_PLAN=GenCopy
 

.github/scripts/pgo-build.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Compile with profiling support
+RUSTFLAGS="-Cprofile-generate=/tmp/$USER/pgo-data" make CONF=linux-x86_64-normal-server-release THIRD_PARTY_HEAP=$PWD/../mmtk-openjdk/openjdk images
+
+# Remove extraneous profiling data
+rm -rf /tmp/$USER/pgo-data/*
+
+# Profile using fop
+MMTK_PLAN=GenImmix MMTK_STRESS_FACTOR=4194304 MMTK_PRECISE_STRESS=false ./build/linux-x86_64-normal-server-release/images/jdk/bin/java -XX:MetaspaceSize=500M -XX:+DisableExplicitGC -XX:-TieredCompilation -Xcomp -XX:+UseThirdPartyHeap -Xms60M -Xmx60M -jar /usr/share/benchmarks/dacapo/dacapo-evaluation-git-6e411f33.jar -n 5 fop
+
+# Merge profiling data
+/opt/rust/toolchains/1.66.1-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/bin/llvm-profdata merge -o /tmp/$USER/pgo-data/merged.profdata /tmp/$USER/pgo-data
+
+# Compile using profiling data
+RUSTFLAGS="-Cprofile-use=/tmp/$USER/pgo-data/merged.profdata -Cllvm-args=-pgo-warn-missing-function" make CONF=linux-x86_64-normal-server-release THIRD_PARTY_HEAP=$PWD/../mmtk-openjdk/openjdk images

CHANGELOG.md

@@ -1,3 +1,12 @@
+0.17.0 (2023-02-17)
+===
+
+* MMTk OpenJDK binding now uses Rust 1.66.1 and MSRV is 1.61.0.
+* Support dynamic heap resizing (enabled when `Xmx` and `Xms` values are different).
+* Remove all inline directives. We rely on Rust compiler and PGO for inline decisions. Add provide a PGO guide.
+* Fix a crash caused by null pointer access if the VM calls `CollectedHeap::soft_ref_policy()`.
+* Update to mmtk-core 0.17.0.
+
 0.16.0 (2022-12-06)
 ===
 

README.md

@@ -15,8 +15,8 @@ Please make sure your dev machine satisfies those prerequisites.
 
 ### Before you continue
 
-The minimal supported Rust version for MMTk-OpenJDK binding is 1.57.0. Make sure your Rust version is higher than this. We test MMTk-OpenJDK
-binding with Rust 1.59.0 (as specified in [`rust-toolchain`](mmtk/rust-toolchain)).
+The minimal supported Rust version for MMTk-OpenJDK binding is 1.61.0. Make sure your Rust version is higher than this. We test MMTk-OpenJDK
+binding with Rust 1.66.1 (as specified in [`rust-toolchain`](mmtk/rust-toolchain)).
 You may also need to use ssh-agent to authenticate with github (see [here](https://github.com/rust-lang/cargo/issues/3487) for more info):
 
 ```console
@@ -126,6 +126,67 @@ $ make CONF=linux-x86_64-normal-server-release THIRD_PARTY_HEAP=$PWD/../mmtk-ope
 
 The output jdk is then found at `./build/linux-x86_64-normal-server-release/images/jdk`.
 
+### Profile-Guided Optimized Build
+
+In order to get the best performance, we recommend using a profile-guided
+optimized (PGO) build. Rust supports [PGO
+builds](https://doc.rust-lang.org/rustc/profile-guided-optimization.html) by
+directly hooking into the LLVM profiling infrastructure. In order to have the
+correct LLVM tools version, you should install the relevant `llvm-tools-preview`
+component using `rustup`:
+
+```console
+$ rustup component add llvm-tools-preview
+```
+
+In this example, we focus on the DaCapo benchmarks and the `GenImmix`
+collector. For best results, it is recommended to profile the workload you are
+interested in measuring. We use `fop` as it is a relatively small benchmark but
+also exercises the GC. In order to best tune our GC performance, we use a
+stress factor of 4 MB in order to trigger more GC events.
+
+First we compile MMTk with profiling support:
+
+```console
+$ RUSTFLAGS="-Cprofile-generate=/tmp/$USER/pgo-data" make CONF=linux-x86_64-normal-server-release THIRD_PARTY_HEAP=$PWD/../mmtk-openjdk/openjdk images
+$ rm -rf /tmp/$USER/pgo-data/*
+```
+We clear the `/tmp/$USER/pgo-data` directory as during compilation, the JVM we
+have created is used in a bootstrap process, resulting in profile data being
+emitted.
+
+We then run `fop` in order to get some profiling data. Note that your location
+for the DaCapo benchmarks may be different:
+
+```bash
+MMTK_PLAN=GenImmix MMTK_STRESS_FACTOR=4194304 MMTK_PRECISE_STRESS=false ./build/linux-x86_64-normal-server-release/images/jdk/bin/java -XX:MetaspaceSize=500M -XX:+DisableExplicitGC -XX:-TieredCompilation -Xcomp -XX:+UseThirdPartyHeap -Xms60M -Xmx60M -jar /usr/share/benchmarks/dacapo/dacapo-evaluation-git-6e411f33.jar -n 5 fop
+```
+
+We have to merge the profiling data into something we can feed into the Rust
+compiler using `llvm-profdata`:
+
+```console
+$ /opt/rust/toolchains/1.66.1-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/bin/llvm-profdata merge -o /tmp/$USER/pgo-data/merged.profdata /tmp/$USER/pgo-data
+```
+
+The location of your version of `llvm-profdata` may be different to what we
+have above. *Make sure to only use a version of `llvm-profdata` that matches
+your Rust version.*
+
+Finally, we build a new image using the profiling data as an input:
+
+```console
+$ RUSTFLAGS="-Cprofile-use=/tmp/$USER/pgo-data/merged.profdata -Cllvm-args=-pgo-warn-missing-function" make CONF=linux-x86_64-normal-server-release THIRD_PARTY_HEAP=$PWD/../mmtk-openjdk/openjdk images
+```
+
+We now have an OpenJDK build under
+`./build/linux-x86_64-normal-server-release/images/jdk` with MMTk that has been
+optimized using PGO.
+
+For ease of use, we have provided an example script which does the above in
+`.github/scripts/pgo-build.sh` that you may adapt for your purposes. Note that
+you may have to change the location of `llvm-profdata`.
+
 ### Location of Mark-bit
 The location of the mark-bit can be specified by the environment variable
 `MARK_IN_HEADER`. By default, the mark-bit is located on the side (in a side