diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0593afd02f3..8ca284180d4 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -40,7 +40,7 @@ on:
platforms:
description: 'Platform(s) to execute on (comma separated, e.g. "linux-x64, macos, aarch64")'
required: true
- default: 'linux-x64, linux-x86-hs, linux-x64-variants, linux-cross-compile, alpine-linux-x64, macos-x64, macos-aarch64, windows-x64, windows-aarch64, docs'
+ default: 'linux-x64, linux-x64-variants, linux-cross-compile, alpine-linux-x64, macos-x64, macos-aarch64, windows-x64, windows-aarch64, docs'
configure-arguments:
description: 'Additional configure arguments'
required: false
@@ -68,7 +68,6 @@ jobs:
EXCLUDED_PLATFORMS: 'alpine-linux-x64'
outputs:
linux-x64: ${{ steps.include.outputs.linux-x64 }}
- linux-x86-hs: ${{ steps.include.outputs.linux-x86-hs }}
linux-x64-variants: ${{ steps.include.outputs.linux-x64-variants }}
linux-cross-compile: ${{ steps.include.outputs.linux-cross-compile }}
alpine-linux-x64: ${{ steps.include.outputs.alpine-linux-x64 }}
@@ -152,7 +151,6 @@ jobs:
}
echo "linux-x64=$(check_platform linux-x64 linux x64)" >> $GITHUB_OUTPUT
- echo "linux-x86-hs=$(check_platform linux-x86-hs linux x86)" >> $GITHUB_OUTPUT
echo "linux-x64-variants=$(check_platform linux-x64-variants variants)" >> $GITHUB_OUTPUT
echo "linux-cross-compile=$(check_platform linux-cross-compile cross-compile)" >> $GITHUB_OUTPUT
echo "alpine-linux-x64=$(check_platform alpine-linux-x64 alpine-linux x64)" >> $GITHUB_OUTPUT
@@ -177,24 +175,6 @@ jobs:
make-arguments: ${{ github.event.inputs.make-arguments }}
if: needs.prepare.outputs.linux-x64 == 'true'
- build-linux-x86-hs:
- name: linux-x86-hs
- needs: prepare
- uses: ./.github/workflows/build-linux.yml
- with:
- platform: linux-x86
- make-target: 'hotspot'
- gcc-major-version: '10'
- gcc-package-suffix: '-multilib'
- apt-architecture: 'i386'
- # Some multilib libraries do not have proper inter-dependencies, so we have to
- # install their dependencies manually.
- apt-extra-packages: 'libfreetype-dev:i386 libtiff-dev:i386 libcupsimage2-dev:i386 libffi-dev:i386'
- extra-conf-options: '--with-target-bits=32 --enable-fallback-linker --enable-libffi-bundling'
- configure-arguments: ${{ github.event.inputs.configure-arguments }}
- make-arguments: ${{ github.event.inputs.make-arguments }}
- if: needs.prepare.outputs.linux-x86-hs == 'true'
-
build-linux-x64-hs-nopch:
name: linux-x64-hs-nopch
needs: prepare
diff --git a/doc/building.html b/doc/building.html
index 63af224584a..cd73863f879 100644
--- a/doc/building.html
+++ b/doc/building.html
@@ -329,8 +329,8 @@ Building on x86
Even for 32-bit builds, it is recommended to use a 64-bit build
machine, and instead create a 32-bit target using
--with-target-bits=32.
-Note: The Windows 32-bit x86 port is deprecated and may be removed in
-a future release.
+Note: The 32-bit x86 port is deprecated and may be removed in a
+future release.
Building on aarch64
At a minimum, a machine with 8 cores is advisable, as well as 8 GB of
RAM. (The more cores to use, the more memory you need.) At least 6 GB of
@@ -393,8 +393,7 @@
Operating System
to list successes or failures of building on different platforms.
Windows
Windows XP is not a supported platform, but all newer Windows should
-be able to build the JDK. (Note: The Windows 32-bit x86 port is
-deprecated and may be removed in a future release.)
+be able to build the JDK.
On Windows, it is important that you pay attention to the
instructions in the Special
Considerations.
diff --git a/doc/building.md b/doc/building.md
index 466e8d7edf8..99bc509dc70 100644
--- a/doc/building.md
+++ b/doc/building.md
@@ -134,8 +134,7 @@ space is required.
Even for 32-bit builds, it is recommended to use a 64-bit build machine, and
instead create a 32-bit target using `--with-target-bits=32`.
-Note: The Windows 32-bit x86 port is deprecated and may be removed in a future
-release.
+Note: The 32-bit x86 port is deprecated and may be removed in a future release.
### Building on aarch64
@@ -191,8 +190,7 @@ on different platforms.
### Windows
Windows XP is not a supported platform, but all newer Windows should be able to
-build the JDK. (Note: The Windows 32-bit x86 port is deprecated and may be
-removed in a future release.)
+build the JDK.
On Windows, it is important that you pay attention to the instructions in the
[Special Considerations](#special-considerations).
diff --git a/make/Docs.gmk b/make/Docs.gmk
index e6a98c4fbd2..fb2726e6dad 100644
--- a/make/Docs.gmk
+++ b/make/Docs.gmk
@@ -107,15 +107,13 @@ JAVA_WARNINGS_ARE_ERRORS ?= -Werror
JAVADOC_OPTIONS := -use -keywords -notimestamp \
-encoding ISO-8859-1 -docencoding UTF-8 -breakiterator \
-splitIndex --system none -javafx --expand-requires transitive \
- --override-methods=summary \
- --no-external-specs-page
+ --override-methods=summary
# The reference options must stay stable to allow for comparisons across the
# development cycle.
REFERENCE_OPTIONS := -XDignore.symbol.file=true -use -keywords -notimestamp \
-encoding ISO-8859-1 -breakiterator -splitIndex --system none \
- -html5 -javafx --expand-requires transitive \
- --no-external-specs-page
+ -html5 -javafx --expand-requires transitive
# Should we add DRAFT stamps to the generated javadoc?
ifeq ($(VERSION_IS_GA), true)
diff --git a/make/Images.gmk b/make/Images.gmk
index 39b00e82e55..053187859dd 100644
--- a/make/Images.gmk
+++ b/make/Images.gmk
@@ -317,27 +317,6 @@ else
endif
CMDS_TARGET_SUBDIR := bin
-# Param 1 - dir to find debuginfo files in
-FindDebuginfoFiles = \
- $(wildcard $(addprefix $1/*, $(DEBUGINFO_SUFFIXES)) \
- $(addprefix $1/*/*, $(DEBUGINFO_SUFFIXES)) \
- $(addprefix $1/*/*/*, $(DEBUGINFO_SUFFIXES)))
-
-# Pick the correct debug info files to copy, either zipped or not.
-ifeq ($(ZIP_EXTERNAL_DEBUG_SYMBOLS), true)
- DEBUGINFO_SUFFIXES += .diz
-else
- DEBUGINFO_SUFFIXES := .debuginfo .pdb .map
- # On Macosx, if debug symbols have not been zipped, find all files inside *.dSYM
- # dirs.
- ifeq ($(call isTargetOs, macosx), true)
- $(call FillFindCache, \
- $(SUPPORT_OUTPUTDIR)/modules_libs $(SUPPORT_OUTPUTDIR)/modules_cmds)
- FindDebuginfoFiles = \
- $(if $(wildcard $1), $(call containing, .dSYM/, $(call FindFiles, $1)))
- endif
-endif
-
# Param 1 - either JDK or JRE
SetupCopyDebuginfo = \
$(foreach m, $(ALL_$1_MODULES), \
diff --git a/make/Main.gmk b/make/Main.gmk
index 152dbf17cd3..ddcc0c7f674 100644
--- a/make/Main.gmk
+++ b/make/Main.gmk
@@ -454,6 +454,18 @@ $(eval $(call SetupTarget, symbols-image, \
TARGET := symbols, \
))
+$(eval $(call SetupTarget, static-launcher, \
+ MAKEFILE := StaticLibs, \
+ TARGET := static-launcher, \
+ DEPS := hotspot-static-libs static-libs, \
+))
+
+$(eval $(call SetupTarget, static-jdk-image, \
+ MAKEFILE := StaticLibs, \
+ TARGET := static-jdk-image, \
+ DEPS := static-exploded-image jdk-image, \
+))
+
$(eval $(call SetupTarget, static-libs-image, \
MAKEFILE := StaticLibsImage, \
TARGET := static-libs-image, \
@@ -1086,9 +1098,9 @@ else
symbols-image: $(LIBS_TARGETS) $(LAUNCHER_TARGETS)
- static-libs-image: hotspot-static-libs $(STATIC_LIBS_TARGETS)
+ static-libs-image: hotspot-static-libs static-libs
- static-libs-graal-image: $(STATIC_LIBS_TARGETS)
+ static-libs-graal-image: static-libs
bootcycle-images: jdk-image
@@ -1254,6 +1266,8 @@ ifeq ($(call isTargetOs, macosx), true)
legacy-images: mac-legacy-jre-bundle
endif
+static-exploded-image: static-launcher exploded-image
+
# These targets build the various documentation images
docs-jdk-image: docs-jdk
docs-javase-image: docs-javase
@@ -1296,7 +1310,7 @@ endif
################################################################################
# all-images builds all our deliverables as images.
-all-images: product-images test-image all-docs-images
+all-images: product-images static-jdk-image test-image all-docs-images
# all-bundles packages all our deliverables as tar.gz bundles.
all-bundles: product-bundles test-bundles docs-bundles static-libs-bundles
@@ -1309,7 +1323,7 @@ ALL_TARGETS += buildtools hotspot hotspot-libs hotspot-static-libs \
create-buildjdk docs-jdk-api docs-javase-api docs-reference-api docs-jdk \
docs-javase docs-reference docs-javadoc mac-bundles product-images legacy-images \
docs-image docs-javase-image docs-reference-image all-docs-images \
- docs-bundles all-docs-bundles test-image all-images \
+ docs-bundles all-docs-bundles test-image all-images static-exploded-image \
all-bundles
################################################################################
diff --git a/make/ModuleWrapper.gmk b/make/ModuleWrapper.gmk
index 14298d25a53..51208432ea0 100644
--- a/make/ModuleWrapper.gmk
+++ b/make/ModuleWrapper.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2014, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -45,6 +45,23 @@ TARGETS :=
# Include the file being wrapped.
include $(MAKEFILE_PREFIX).gmk
+ifeq ($(MAKEFILE_PREFIX), Lib)
+ # We need to keep track of what libraries are generated/needed by this
+ # module. This information is required when doing static linking, to know
+ # which static library files to include. The variable $(MODULE)_INCLUDED_LIBS is
+ # added to for each call to SetupJdkLibrary. The file module-included-libs.txt is then
+ # read in StaticLibs.gmk.
+ ifneq ($($(MODULE)_INCLUDED_LIBS), )
+ LIBLIST := $(SUPPORT_OUTPUTDIR)/modules_static-libs/$(MODULE)/module-included-libs.txt
+
+ $(LIBLIST): $(TARGETS)
+ $(call MakeDir, $(@D))
+ $(ECHO) $($(MODULE)_INCLUDED_LIBS) > $@
+
+ TARGETS += $(LIBLIST)
+ endif
+endif
+
# Setup copy rules from the modules directories to the jdk image directory.
ifeq ($(call isTargetOs, windows), true)
TO_BIN_FILTER := %$(SHARED_LIBRARY_SUFFIX) %.diz %.pdb %.map
diff --git a/make/StaticLibs.gmk b/make/StaticLibs.gmk
new file mode 100644
index 00000000000..78918c456ee
--- /dev/null
+++ b/make/StaticLibs.gmk
@@ -0,0 +1,191 @@
+#
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+default: all
+
+include $(SPEC)
+include MakeBase.gmk
+
+include CopyFiles.gmk
+include Modules.gmk
+include modules/LauncherCommon.gmk
+
+################################################################################
+#
+# Create the static java launcher
+#
+################################################################################
+
+STATIC_JDK_IMAGE_DIR := $(IMAGES_OUTPUTDIR)/static-jdk
+STATIC_LAUNCHER_OUTPUT_DIR := $(SUPPORT_OUTPUTDIR)/static-native/launcher
+HOTSPOT_STATIC_LIB_PATH := $(HOTSPOT_OUTPUTDIR)/*/libjvm/objs/static
+
+ifneq ($(word 2, $(wildcard $(HOTSPOT_STATIC_LIB_PATH))), )
+ $(error Cannot perform static linking when building more than one JVM library)
+endif
+
+# Find all modules with static libraries
+STATIC_LIB_MODULES := $(patsubst $(SUPPORT_OUTPUTDIR)/modules_static-libs/%, \
+ %, $(wildcard $(SUPPORT_OUTPUTDIR)/modules_static-libs/*))
+
+# Filter out known broken libraries. This is a temporary measure until
+# proper support for these libraries can be provided.
+ifeq ($(call isTargetOs, linux), true)
+ # libsplashscreen has a name conflict with libawt in the function
+ # BitmapToYXBandedRectangles, so we exclude it for now.
+ BROKEN_STATIC_LIBS += splashscreen
+else ifeq ($(call isTargetOs, macosx), true)
+ # libosxsecurity has a name conflict with libosxapp in the function
+ # JavaStringToNSString, so we exclude it for now.
+ BROKEN_STATIC_LIBS += osxsecurity
+else ifeq ($(call isTargetOs, windows), true)
+ # libsplashscreen has a name conflict with libawt in the function
+ # BitmapToYXBandedRectangles, so we exclude it for now.
+ BROKEN_STATIC_LIBS += splashscreen
+ # libsspi_bridge has name conflicts with sunmscapi
+ BROKEN_STATIC_LIBS += sspi_bridge
+ # These libs define DllMain which conflict with Hotspot
+ BROKEN_STATIC_LIBS += awt dt_shmem dt_socket
+ # These libs are dependent on any of the above disabled libs
+ BROKEN_STATIC_LIBS += fontmanager jawt lcms net nio
+endif
+
+$(foreach module, $(STATIC_LIB_MODULES), \
+ $(eval LIBS_$(module) := $(filter-out $(BROKEN_STATIC_LIBS), $(shell cat \
+ $(SUPPORT_OUTPUTDIR)/modules_static-libs/$(module)/module-included-libs.txt))) \
+)
+
+STATIC_LIB_FILES := $(foreach module, $(STATIC_LIB_MODULES), \
+ $(foreach lib, $(LIBS_$(module)), \
+ $(SUPPORT_OUTPUTDIR)/native/$(module)/lib$(lib)/static/$(LIBRARY_PREFIX)$(lib)$(STATIC_LIBRARY_SUFFIX)))
+
+# Add Hotspot
+STATIC_LIB_FILES += $(wildcard $(HOTSPOT_STATIC_LIB_PATH)/$(LIBRARY_PREFIX)jvm$(STATIC_LIBRARY_SUFFIX))
+
+# Figure out what external libraries are required to link these static JDK
+# libraries.
+LIB_FLAGS_FILES := $(addsuffix .lib-flags.txt, $(STATIC_LIB_FILES))
+
+# Gather the lib flags from all individual libraries. There are many duplicates,
+# so sort and just keep unique instances. On macOS, a common pattern is
+# "-framework FooFramework", so we must make sure we keep the two words together.
+EXTERNAL_LIBS := $(strip $(shell $(CAT) $(LIB_FLAGS_FILES) | \
+ $(SED) -e 's/-framework /-framework_/g' | $(TR) ' ' '\n' | $(SORT) -u | \
+ $(SED) -e 's/-framework_/-framework /g'))
+
+ifeq ($(call isTargetOs, macosx), true)
+ STATIC_LIBS := $(addprefix -force_load$(SPACE), $(STATIC_LIB_FILES))
+ STANDARD_LIBS += -lstdc++
+else ifeq ($(call isTargetOs, linux), true)
+ STATIC_LIBS := -Wl,--export-dynamic -Wl,--whole-archive $(STATIC_LIB_FILES) -Wl,--no-whole-archive
+ STANDARD_LIBS := -l:libstdc++.a
+else ifeq ($(call isTargetOs, windows), true)
+ STATIC_LIBS := $(addprefix -wholearchive:, $(STATIC_LIB_FILES))
+else
+ $(error Unsupported platform)
+endif
+
+$(eval $(call SetupBuildLauncher, java, \
+ CFLAGS := -DEXPAND_CLASSPATH_WILDCARDS -DENABLE_ARG_FILES, \
+ EXTRA_RCFLAGS := $(JAVA_RCFLAGS), \
+ VERSION_INFO_RESOURCE := $(JAVA_VERSION_INFO_RESOURCE), \
+ OPTIMIZATION := HIGH, \
+ STATIC_LAUNCHER := true, \
+ LDFLAGS := $(LDFLAGS_STATIC_JDK), \
+ LIBS := $(STATIC_LIBS) $(EXTERNAL_LIBS) $(STANDARD_LIBS), \
+ OUTPUT_DIR := $(STATIC_LAUNCHER_OUTPUT_DIR), \
+ OBJECT_DIR := $(STATIC_LAUNCHER_OUTPUT_DIR), \
+))
+
+$(java): $(STATIC_LIB_FILES)
+
+TARGETS += $(java)
+
+JAVA_LAUNCHER := $(BUILD_LAUNCHER_java_TARGET)
+
+static-launcher: $(java)
+
+################################################################################
+#
+# Create the static-jdk image with the statically built java launcher
+#
+################################################################################
+
+# Until we get proper support in jlink for generating an image with static
+# builds, we need to create the image ourselves. We base it on a normal
+# dynamically linked JDK image.
+
+# All these files/dirs should be copied as-is
+JDK_IMAGE_COPY_FILES := $(addprefix $(JDK_IMAGE_DIR)/, conf demo include jmods \
+ legal man/man1/java.1 release README)
+
+# We need to copy some files from lib, but not the dynamic libraries themselves
+ALL_LIB_FILES := $(call FindFiles, $(JDK_IMAGE_DIR)/lib)
+
+# Remove all dynamic libraries from the list
+JDK_IMAGE_COPY_LIB_FILES := $(filter-out %$(SHARED_LIBRARY_SUFFIX), $(ALL_LIB_FILES))
+# Remove all debug files from the list
+ifeq ($(call isTargetOs, macosx), true)
+ JDK_IMAGE_COPY_LIB_FILES := $(call not-containing, .dSYM, $(JDK_IMAGE_COPY_LIB_FILES))
+else
+ JDK_IMAGE_COPY_LIB_FILES := $(filter-out %.debuginfo %.pdb %.map, $(JDK_IMAGE_COPY_LIB_FILES))
+endif
+
+static-jdk-info:
+ $(call LogWarn, Creating static-jdk image)
+
+$(eval $(call SetupCopyFiles, copy-from-jdk-image, \
+ SRC := $(JDK_IMAGE_DIR), \
+ DEST := $(STATIC_JDK_IMAGE_DIR), \
+ FILES := $(call FindFiles, $(JDK_IMAGE_COPY_FILES)) \
+ $(JDK_IMAGE_COPY_LIB_FILES), \
+))
+
+TARGETS += $(copy-from-jdk-image)
+
+$(copy-from-jdk-image): | static-jdk-info
+
+$(eval $(call SetupCopyFiles, copy-static-launcher, \
+ FILES := $(JAVA_LAUNCHER), \
+ DEST := $(STATIC_JDK_IMAGE_DIR)/bin, \
+))
+
+TARGETS += $(copy-static-launcher)
+
+$(eval $(call SetupCopyFiles, copy-static-launcher-debuginfo, \
+ SRC := $(STATIC_LAUNCHER_OUTPUT_DIR), \
+ DEST := $(STATIC_JDK_IMAGE_DIR)/bin, \
+ FILES := $(call FindDebuginfoFiles, $(STATIC_LAUNCHER_OUTPUT_DIR)), \
+))
+
+TARGETS += $(copy-static-launcher-debuginfo)
+
+static-jdk-image: $(copy-from-jdk-image) $(copy-static-launcher) $(copy-static-launcher-debuginfo)
+
+TARGETS += static-jdk-image
+
+all: $(TARGETS)
+
+.PHONY: all static-launcher static-jdk-image
diff --git a/make/autoconf/boot-jdk.m4 b/make/autoconf/boot-jdk.m4
index d729012ad6a..d39e6e75a94 100644
--- a/make/autoconf/boot-jdk.m4
+++ b/make/autoconf/boot-jdk.m4
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -470,7 +470,7 @@ AC_DEFUN_ONCE([BOOTJDK_SETUP_BOOT_JDK_ARGUMENTS],
# Maximum amount of heap memory.
JVM_HEAP_LIMIT_32="768"
# Running a 64 bit JVM allows for and requires a bigger heap
- JVM_HEAP_LIMIT_64="1600"
+ JVM_HEAP_LIMIT_64="2048"
JVM_HEAP_LIMIT_GLOBAL=`expr $MEMORY_SIZE / 2`
if test "$JVM_HEAP_LIMIT_GLOBAL" -lt "$JVM_HEAP_LIMIT_32"; then
JVM_HEAP_LIMIT_32=$JVM_HEAP_LIMIT_GLOBAL
diff --git a/make/autoconf/flags-ldflags.m4 b/make/autoconf/flags-ldflags.m4
index 851b3b7a0ef..ffb1f0d6e19 100644
--- a/make/autoconf/flags-ldflags.m4
+++ b/make/autoconf/flags-ldflags.m4
@@ -192,18 +192,23 @@ AC_DEFUN([FLAGS_SETUP_LDFLAGS_CPU_DEP],
# Export variables according to old definitions, prefix with $2 if present.
LDFLAGS_JDK_COMMON="$BASIC_LDFLAGS $BASIC_LDFLAGS_JDK_ONLY \
$OS_LDFLAGS $DEBUGLEVEL_LDFLAGS_JDK_ONLY ${$2EXTRA_LDFLAGS}"
- $2LDFLAGS_JDKLIB="$LDFLAGS_JDK_COMMON $BASIC_LDFLAGS_JDK_LIB_ONLY \
+ $2LDFLAGS_JDKLIB="$LDFLAGS_JDK_COMMON \
$SHARED_LIBRARY_FLAGS $REPRODUCIBLE_LDFLAGS $FILE_MACRO_LDFLAGS"
$2LDFLAGS_JDKEXE="$LDFLAGS_JDK_COMMON $EXECUTABLE_LDFLAGS \
${$1_CPU_EXECUTABLE_LDFLAGS} $REPRODUCIBLE_LDFLAGS $FILE_MACRO_LDFLAGS"
+ $2LDFLAGS_STATIC_JDK="$BASIC_LDFLAGS $BASIC_LDFLAGS_JVM_ONLY \
+ $OS_LDFLAGS ${$2EXTRA_LDFLAGS} $REPRODUCIBLE_LDFLAGS $FILE_MACRO_LDFLAGS"
+
$2JVM_LDFLAGS="$BASIC_LDFLAGS $BASIC_LDFLAGS_JVM_ONLY $OS_LDFLAGS $OS_LDFLAGS_JVM_ONLY \
- $DEBUGLEVEL_LDFLAGS $DEBUGLEVEL_LDFLAGS_JVM_ONLY $BASIC_LDFLAGS_ONLYCXX \
+ $DEBUGLEVEL_LDFLAGS $DEBUGLEVEL_LDFLAGS_JVM_ONLY \
${$1_CPU_LDFLAGS} ${$1_CPU_LDFLAGS_JVM_ONLY} ${$2EXTRA_LDFLAGS} \
$REPRODUCIBLE_LDFLAGS $FILE_MACRO_LDFLAGS"
AC_SUBST($2LDFLAGS_JDKLIB)
AC_SUBST($2LDFLAGS_JDKEXE)
+ AC_SUBST($2LDFLAGS_STATIC_JDK)
+
AC_SUBST($2JVM_LDFLAGS)
])
diff --git a/make/autoconf/jdk-options.m4 b/make/autoconf/jdk-options.m4
index 4da91a64324..fa8018ccd9f 100644
--- a/make/autoconf/jdk-options.m4
+++ b/make/autoconf/jdk-options.m4
@@ -384,6 +384,7 @@ AC_DEFUN_ONCE([JDKOPT_SETUP_CODE_COVERAGE],
CXXFLAGS_JDKEXE="$CXXFLAGS_JDKEXE $GCOV_CFLAGS"
LDFLAGS_JDKLIB="$LDFLAGS_JDKLIB $GCOV_LDFLAGS"
LDFLAGS_JDKEXE="$LDFLAGS_JDKEXE $GCOV_LDFLAGS"
+ LDFLAGS_STATIC_JDK="$LDFLAGS_STATIC_JDK $GCOV_LDFLAGS"
])
AC_SUBST(GCOV_ENABLED)
@@ -478,6 +479,7 @@ AC_DEFUN_ONCE([JDKOPT_SETUP_ADDRESS_SANITIZER],
CXXFLAGS_JDKEXE="$CXXFLAGS_JDKEXE $ASAN_CFLAGS"
LDFLAGS_JDKLIB="$LDFLAGS_JDKLIB $ASAN_LDFLAGS"
LDFLAGS_JDKEXE="$LDFLAGS_JDKEXE $ASAN_LDFLAGS"
+ LDFLAGS_STATIC_JDK="$LDFLAGS_STATIC_JDK $ASAN_LDFLAGS"
])
AC_SUBST(ASAN_ENABLED)
])
@@ -511,6 +513,7 @@ AC_DEFUN_ONCE([JDKOPT_SETUP_LEAK_SANITIZER],
CXXFLAGS_JDKEXE="$CXXFLAGS_JDKEXE $LSAN_CFLAGS"
LDFLAGS_JDKLIB="$LDFLAGS_JDKLIB $LSAN_LDFLAGS"
LDFLAGS_JDKEXE="$LDFLAGS_JDKEXE $LSAN_LDFLAGS"
+ LDFLAGS_STATIC_JDK="$LDFLAGS_STATIC_JDK $LSAN_LDFLAGS"
])
AC_SUBST(LSAN_ENABLED)
])
@@ -553,6 +556,7 @@ AC_DEFUN_ONCE([JDKOPT_SETUP_UNDEFINED_BEHAVIOR_SANITIZER],
CXXFLAGS_JDKEXE="$CXXFLAGS_JDKEXE $UBSAN_CFLAGS"
LDFLAGS_JDKLIB="$LDFLAGS_JDKLIB $UBSAN_LDFLAGS"
LDFLAGS_JDKEXE="$LDFLAGS_JDKEXE $UBSAN_LDFLAGS"
+ LDFLAGS_STATIC_JDK="$LDFLAGS_STATIC_JDK $UBSAN_LDFLAGS"
])
if test "x$UBSAN_ENABLED" = xfalse; then
UBSAN_CFLAGS=""
diff --git a/make/autoconf/platform.m4 b/make/autoconf/platform.m4
index 3f977058a51..937d8c37927 100644
--- a/make/autoconf/platform.m4
+++ b/make/autoconf/platform.m4
@@ -666,14 +666,17 @@ AC_DEFUN([PLATFORM_CHECK_DEPRECATION],
[
AC_ARG_ENABLE(deprecated-ports, [AS_HELP_STRING([--enable-deprecated-ports@<:@=yes/no@:>@],
[Suppress the error when configuring for a deprecated port @<:@no@:>@])])
- # if test "x$OPENJDK_TARGET_CPU" = xx86; then
- # if test "x$enable_deprecated_ports" = "xyes"; then
- # AC_MSG_WARN([The x86 port is deprecated and may be removed in a future release.])
- # else
- # AC_MSG_ERROR(m4_normalize([The 32-bit x86 port is deprecated and may be removed in a future release.
- # Use --enable-deprecated-ports=yes to suppress this error.]))
- # fi
- # fi
+ # Unfortunately, variants have not been parsed yet, so we have to check the configure option
+ # directly. Allow only the directly specified Zero variant, treat any other mix as containing
+ # something non-Zero.
+ if test "x$OPENJDK_TARGET_CPU" = xx86 && test "x$with_jvm_variants" != xzero; then
+ if test "x$enable_deprecated_ports" = "xyes"; then
+ AC_MSG_WARN([The 32-bit x86 port is deprecated and may be removed in a future release.])
+ else
+ AC_MSG_ERROR(m4_normalize([The 32-bit x86 port is deprecated and may be removed in a future release.
+ Use --enable-deprecated-ports=yes to suppress this error.]))
+ fi
+ fi
])
AC_DEFUN_ONCE([PLATFORM_SETUP_OPENJDK_BUILD_OS_VERSION],
diff --git a/make/autoconf/spec.gmk.template b/make/autoconf/spec.gmk.template
index 17b7b555f37..c67527ae7f1 100644
--- a/make/autoconf/spec.gmk.template
+++ b/make/autoconf/spec.gmk.template
@@ -563,6 +563,9 @@ LDFLAGS_JDKLIB := @LDFLAGS_JDKLIB@
# LDFLAGS used to link the jdk native launchers (C-code)
LDFLAGS_JDKEXE := @LDFLAGS_JDKEXE@
+# LDFLAGS used to link the static jdk library
+LDFLAGS_STATIC_JDK := @LDFLAGS_STATIC_JDK@
+
# LDFLAGS specific to C++ linking.
LDFLAGS_CXX_JDK := @LDFLAGS_CXX_JDK@
diff --git a/make/autoconf/util.m4 b/make/autoconf/util.m4
index 5a6142d5092..aa9662e2100 100644
--- a/make/autoconf/util.m4
+++ b/make/autoconf/util.m4
@@ -575,7 +575,7 @@ AC_DEFUN([UTIL_CHECK_TYPE_directory],
if test "[x]ARG_CHECK_FOR_FILES" != "x:"; then
for file in ARG_CHECK_FOR_FILES; do
- found_files=$($ECHO $(ls $1/$file 2> /dev/null))
+ found_files=$($ECHO $($LS -d $1/$file 2> /dev/null))
if test "x$found_files" = x; then
FAILURE="Directory $1 does not contain $file"
break
diff --git a/make/common/FileUtils.gmk b/make/common/FileUtils.gmk
index a1bf917b599..b01bc6a7296 100644
--- a/make/common/FileUtils.gmk
+++ b/make/common/FileUtils.gmk
@@ -331,3 +331,26 @@ ifeq ($(DISABLE_CACHE_FIND), true)
else
FindFiles = $(CacheFindFiles)
endif
+
+# Find native debuginfo files in a directory
+#
+# Param 1 - dir to find debuginfo files in
+FindDebuginfoFiles = \
+ $(wildcard $(addprefix $1/*, $(DEBUGINFO_SUFFIXES)) \
+ $(addprefix $1/*/*, $(DEBUGINFO_SUFFIXES)) \
+ $(addprefix $1/*/*/*, $(DEBUGINFO_SUFFIXES)))
+
+# Pick the correct debug info files to copy, either zipped or not.
+ifeq ($(ZIP_EXTERNAL_DEBUG_SYMBOLS), true)
+ DEBUGINFO_SUFFIXES += .diz
+else
+ DEBUGINFO_SUFFIXES := .debuginfo .pdb .map
+ # On Macosx, if debug symbols have not been zipped, find all files inside *.dSYM
+ # dirs.
+ ifeq ($(call isTargetOs, macosx), true)
+ $(call FillFindCache, \
+ $(SUPPORT_OUTPUTDIR)/modules_libs $(SUPPORT_OUTPUTDIR)/modules_cmds)
+ FindDebuginfoFiles = \
+ $(if $(wildcard $1), $(call containing, .dSYM/, $(call FindFiles, $1)))
+ endif
+endif
diff --git a/make/common/JdkNativeCompilation.gmk b/make/common/JdkNativeCompilation.gmk
index 52b6dea6725..ca0f1429c61 100644
--- a/make/common/JdkNativeCompilation.gmk
+++ b/make/common/JdkNativeCompilation.gmk
@@ -275,6 +275,8 @@ JDK_RCFLAGS=$(RCFLAGS) \
# and EXTRA_HEADER_DIRS will be added.
# JDK_LIBS_ or JDK_LIBS_ -- additional JDK_LIBS for the given OS
# or OS type only
+# ONLY_EXPORTED -- if true, this library will be flagged as not
+# to be included for this module when building static libs
# EXTRA_RCFLAGS -- additional RCFLAGS to append.
# RC_FILEDESC -- override the default FILEDESC for Windows version.rc
# DEFAULT_LIBCXX -- if false, do not add LIBCXX to LIBS for C++ compilations
@@ -303,6 +305,15 @@ define SetupJdkNativeCompilationBody
$1_RC_FTYPE := 0x2L
endif
+ ifneq ($$(MODULE), )
+ # Record the fact that this native library is part of the current module
+ # (unless told otherwise). This variable stores information about all
+ # created libraries, and is read by ModuleWrapper.
+ ifneq ($$($1_ONLY_EXPORTED), true)
+ $$(MODULE)_INCLUDED_LIBS += $$($1_NAME)
+ endif
+ endif
+
ifeq ($$($1_OUTPUT_DIR), )
ifneq ($$(MODULE), )
ifeq ($$($1_TYPE), STATIC_LIBRARY)
@@ -422,10 +433,10 @@ define SetupJdkNativeCompilationBody
ifneq ($$($1_DEFAULT_LDFLAGS), false)
ifeq ($$($1_TYPE), EXECUTABLE)
# Set the default flags first to be able to override
- $1_LDFLAGS := $$(filter-out $$($1_LDFLAGS_FILTER_OUT), $$(LDFLAGS_JDKEXE)) $$($1_LDFLAGS)
+ $1_LDFLAGS := $$(filter-out $$($1_LDFLAGS_FILTER_OUT), $$(LDFLAGS_JDKEXE) $$($1_LDFLAGS))
else
# Set the default flags first to be able to override
- $1_LDFLAGS := $$(filter-out $$($1_LDFLAGS_FILTER_OUT), $$(LDFLAGS_JDKLIB)) $$($1_LDFLAGS)
+ $1_LDFLAGS := $$(filter-out $$($1_LDFLAGS_FILTER_OUT), $$(LDFLAGS_JDKLIB) $$($1_LDFLAGS))
endif
endif
diff --git a/make/common/modules/LauncherCommon.gmk b/make/common/modules/LauncherCommon.gmk
index b522df076f3..77f39457b4c 100644
--- a/make/common/modules/LauncherCommon.gmk
+++ b/make/common/modules/LauncherCommon.gmk
@@ -62,6 +62,7 @@ JAVA_MANIFEST := $(TOPDIR)/src/java.base/windows/native/launcher/java.manifest
# OPTIMIZATION Override default optimization level (LOW)
# OUTPUT_DIR Override default output directory
# VERSION_INFO_RESOURCE Override default Windows resource file
+# STATIC_LAUNCHER If true, will use settings for building a static launcher
SetupBuildLauncher = $(NamedParamsMacroTemplate)
define SetupBuildLauncherBody
# Setup default values (unless overridden)
@@ -120,6 +121,15 @@ define SetupBuildLauncherBody
$1_EXTRA_FILES += $(TOPDIR)/make/data/lsan/lsan_default_options.c
endif
+ ifneq ($$($1_STATIC_LAUNCHER), true)
+ $1_JDK_LIBS := java.base:libjli
+ $1_JDK_LIBS_windows := java.base:libjava
+ else
+ ifneq ($(findstring $(TOOLCHAIN_TYPE), gcc clang), )
+ $1_LDFLAGS_FILTER_OUT := -Wl$(COMMA)--exclude-libs$(COMMA)ALL
+ endif
+ endif
+
##############################################################################
## Build launcher "$1"
##############################################################################
@@ -140,8 +150,9 @@ define SetupBuildLauncherBody
LDFLAGS := $$($1_LDFLAGS), \
LDFLAGS_linux := $$(call SET_EXECUTABLE_ORIGIN,/../lib), \
LDFLAGS_macosx := $$(call SET_EXECUTABLE_ORIGIN,/../lib), \
- JDK_LIBS := java.base:libjli, \
- JDK_LIBS_windows := java.base:libjava, \
+ LDFLAGS_FILTER_OUT := $$($1_LDFLAGS_FILTER_OUT), \
+ JDK_LIBS := $$($1_JDK_LIBS), \
+ JDK_LIBS_windows := $$($1_JDK_LIBS_windows), \
LIBS := $$($1_LIBS), \
LIBS_unix := $(LIBZ_LIBS), \
LIBS_linux := $(LIBDL) -lpthread, \
@@ -150,6 +161,7 @@ define SetupBuildLauncherBody
-framework Cocoa \
-framework Security, \
OUTPUT_DIR := $$($1_OUTPUT_DIR), \
+ OBJECT_DIR := $$($1_OBJECT_DIR), \
VERSIONINFO_RESOURCE := $$($1_VERSION_INFO_RESOURCE), \
EXTRA_RCFLAGS := $$($1_EXTRA_RCFLAGS), \
MANIFEST := $(JAVA_MANIFEST), \
diff --git a/make/common/native/Link.gmk b/make/common/native/Link.gmk
index 23977e954ca..1461f7302dc 100644
--- a/make/common/native/Link.gmk
+++ b/make/common/native/Link.gmk
@@ -119,6 +119,7 @@ define CreateStaticLibrary
$(if $$($1_LINK_OBJS_RELATIVE), $$(CD) $$(OUTPUTDIR) ; ) \
$$($1_AR) $$(ARFLAGS) -r -cs $$($1_TARGET) \
$$($1_AR_OBJ_ARG) $$($1_RES))
+ $$(ECHO) $$(strip $$($1_LIBS) $$($1_EXTRA_LIBS)) > $$($1_TARGET).lib-flags.txt
endef
################################################################################
diff --git a/make/common/native/LinkMicrosoft.gmk b/make/common/native/LinkMicrosoft.gmk
index cb457034a7e..7c895a9507d 100644
--- a/make/common/native/LinkMicrosoft.gmk
+++ b/make/common/native/LinkMicrosoft.gmk
@@ -54,7 +54,8 @@ define CreateStaticLibraryMicrosoft
$$(call MakeDir, $$($1_OUTPUT_DIR) $$($1_SYMBOLS_DIR))
$$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_run_lib, \
$$($1_LIB) -nologo $$(LIBFLAGS) -out:$$($1_TARGET) \
- $$($1_LD_OBJ_ARG) $$($1_RES))
+ $$($1_LD_OBJ_ARG))
+ $$(ECHO) $$(strip $$($1_LIBS) $$($1_EXTRA_LIBS)) > $$($1_TARGET).lib-flags.txt
endef
################################################################################
diff --git a/make/conf/jib-profiles.js b/make/conf/jib-profiles.js
index 25cf15c5f3b..0d906acdfcb 100644
--- a/make/conf/jib-profiles.js
+++ b/make/conf/jib-profiles.js
@@ -241,7 +241,7 @@ var getJibProfilesCommon = function (input, data) {
// List of the main profile names used for iteration
common.main_profile_names = [
"linux-x64", "linux-x86", "macosx-x64", "macosx-aarch64",
- "windows-x64", "windows-x86", "windows-aarch64",
+ "windows-x64", "windows-aarch64",
"linux-aarch64", "linux-arm32", "linux-ppc64le", "linux-s390x",
"linux-riscv64"
];
@@ -465,15 +465,6 @@ var getJibProfilesProfiles = function (input, common, data) {
configure_args: concat(common.configure_args_64bit),
},
- "windows-x86": {
- target_os: "windows",
- target_cpu: "x86",
- build_cpu: "x64",
- dependencies: ["devkit", "gtest"],
- configure_args: concat(common.configure_args_32bit,
- "--enable-deprecated-ports"),
- },
-
"windows-aarch64": {
target_os: "windows",
target_cpu: "aarch64",
@@ -716,10 +707,6 @@ var getJibProfilesProfiles = function (input, common, data) {
platform: "windows-x64",
jdk_suffix: "zip",
},
- "windows-x86": {
- platform: "windows-x86",
- jdk_suffix: "zip",
- },
"windows-aarch64": {
platform: "windows-aarch64",
jdk_suffix: "zip",
diff --git a/make/hotspot/lib/JvmFeatures.gmk b/make/hotspot/lib/JvmFeatures.gmk
index b94031515f7..09a48508eff 100644
--- a/make/hotspot/lib/JvmFeatures.gmk
+++ b/make/hotspot/lib/JvmFeatures.gmk
@@ -174,6 +174,12 @@ ifeq ($(call check-jvm-feature, link-time-opt), true)
-fno-fat-lto-objects
JVM_LDFLAGS_FEATURES += $(CXX_O_FLAG_HIGHEST_JVM) -flto=auto \
-fuse-linker-plugin -fno-strict-aliasing
+ else ifeq ($(call isCompiler, clang), true)
+ JVM_CFLAGS_FEATURES += -flto -fno-strict-aliasing
+ ifeq ($(call isBuildOs, aix), true)
+ JVM_CFLAGS_FEATURES += -ffat-lto-objects
+ endif
+ JVM_LDFLAGS_FEATURES += $(CXX_O_FLAG_HIGHEST_JVM) -flto -fno-strict-aliasing
else ifeq ($(call isCompiler, microsoft), true)
JVM_CFLAGS_FEATURES += -GL
JVM_LDFLAGS_FEATURES += -LTCG:INCREMENTAL
diff --git a/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java b/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java
index 8865e3908ae..55dd6a8d6ad 100644
--- a/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java
+++ b/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java
@@ -33,6 +33,7 @@
import java.time.*;
import java.util.*;
import java.util.ResourceBundle.Control;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
@@ -1242,7 +1243,8 @@ private static Stream zidMapEntry() {
private static Stream tzDataLinkEntry() {
try {
return Files.walk(Paths.get(tzDataDir), 1)
- .filter(p -> !Files.isDirectory(p))
+ .filter(p -> p.toFile().isFile())
+ .filter(p -> p.getFileName().toString().matches("africa|antarctica|asia|australasia|backward|etcetera|europe|northamerica|southamerica"))
.flatMap(CLDRConverter::extractLinks)
.sorted();
} catch (IOException e) {
@@ -1273,8 +1275,27 @@ private static Stream extractLinks(Path tzFile) {
// Note: the entries are alphabetically sorted, *except* the "world" region
// code, i.e., "001". It should be the last entry for the same windows time
// zone name entries. (cf. TimeZone_md.c)
+ //
+ // The default entries from CLDR's windowsZones.xml file can be modified
+ // with /tzmappings.override where mapping overrides
+ // can be specified.
+ private static Pattern OVERRIDE_PATTERN = Pattern.compile("(?([^:]+:[^:]+)):(?[^:]+):");
private static void generateWindowsTZMappings() throws Exception {
Files.createDirectories(Paths.get(DESTINATION_DIR, "windows", "conf"));
+ var override = Path.of(tzDataDir, "tzmappings.override");
+ if (override.toFile().exists()) {
+ Files.readAllLines(override).stream()
+ .map(String::trim)
+ .filter(o -> !o.isBlank() && !o.startsWith("#"))
+ .forEach(o -> {
+ var m = OVERRIDE_PATTERN.matcher(o);
+ if (m.matches()) {
+ handlerWinZones.put(m.group("win"), m.group("java"));
+ } else {
+ System.out.printf("Unrecognized tzmappings override: %s. Ignored%n", o);
+ }
+ });
+ }
Files.write(Paths.get(DESTINATION_DIR, "windows", "conf", "tzmappings"),
handlerWinZones.keySet().stream()
.filter(k -> k.endsWith(":001") ||
diff --git a/make/modules/java.desktop/lib/AwtLibraries.gmk b/make/modules/java.desktop/lib/AwtLibraries.gmk
index 90eda1c4e64..5ba7c819008 100644
--- a/make/modules/java.desktop/lib/AwtLibraries.gmk
+++ b/make/modules/java.desktop/lib/AwtLibraries.gmk
@@ -168,10 +168,16 @@ ifeq ($(call isTargetOs, windows macosx), false)
# static libraries cause linking errors due to duplicate symbols.
LIBAWT_HEADLESS_STATIC_EXCLUDE_OBJS := systemScale.o
+ ifneq ($(ENABLE_HEADLESS_ONLY), true)
+ # We cannot link with both awt_headless and awt_xawt at the same time
+ LIBAWT_HEADLESS_ONLY_EXPORTED := true
+ endif
+
$(eval $(call SetupJdkLibrary, BUILD_LIBAWT_HEADLESS, \
NAME := awt_headless, \
EXTRA_SRC := $(LIBAWT_HEADLESS_EXTRA_SRC), \
EXCLUDES := medialib, \
+ ONLY_EXPORTED := $(LIBAWT_HEADLESS_ONLY_EXPORTED), \
OPTIMIZATION := LOW, \
CFLAGS := -DHEADLESS=true $(CUPS_CFLAGS) $(FONTCONFIG_CFLAGS) \
$(X_CFLAGS), \
@@ -308,6 +314,8 @@ ifeq ($(call isTargetOs, macosx), true)
LIBAWT_LWAWT_EXCLUDE_FILES := fontpath.c awt_Font.c X11Color.c
LIBAWT_LWAWT_EXCLUDES := $(TOPDIR)/src/$(MODULE)/unix/native/common/awt/medialib
+ LIBAWT_LWAWT_STATIC_EXCLUDE_OBJS := systemScale.o
+
$(eval $(call SetupJdkLibrary, BUILD_LIBAWT_LWAWT, \
NAME := awt_lwawt, \
EXTRA_SRC := $(LIBAWT_LWAWT_EXTRA_SRC), \
@@ -346,6 +354,7 @@ ifeq ($(call isTargetOs, macosx), true)
-framework OpenGL \
-framework QuartzCore \
-framework Security, \
+ STATIC_LIB_EXCLUDE_OBJS := $(LIBAWT_LWAWT_STATIC_EXCLUDE_OBJS), \
))
TARGETS += $(BUILD_LIBAWT_LWAWT)
diff --git a/make/modules/java.desktop/lib/ClientLibraries.gmk b/make/modules/java.desktop/lib/ClientLibraries.gmk
index 31323021780..87f6e2e5099 100644
--- a/make/modules/java.desktop/lib/ClientLibraries.gmk
+++ b/make/modules/java.desktop/lib/ClientLibraries.gmk
@@ -155,6 +155,9 @@ ifeq ($(ENABLE_HEADLESS_ONLY), false)
ifeq ($(USE_EXTERNAL_LIBJPEG), false)
LIBSPLASHSCREEN_EXTRA_SRC += libjavajpeg
+ LIBJAVA_JPEG_OBJS := $(sort $(patsubst %.c,%.o, $(filter-out imageioJPEG.c, \
+ $(notdir $(wildcard $(TOPDIR)/src/java.desktop/share/native/libjavajpeg/*.c)))))
+ LIBSPLASHSCREEN_STATIC_LIB_EXCLUDE_OBJS += $(LIBJAVA_JPEG_OBJS)
endif
ifeq ($(USE_EXTERNAL_LIBPNG), false)
@@ -165,6 +168,10 @@ ifeq ($(ENABLE_HEADLESS_ONLY), false)
ifeq ($(USE_EXTERNAL_LIBZ), false)
LIBSPLASHSCREEN_EXTRA_SRC += java.base:libzip/zlib
+ LIBZIP_SRC_PATH := $(TOPDIR)/src/java.base/share/native/libzip
+ LIBZIP_OBJS := $(sort $(patsubst %.c,%.o, $(notdir \
+ $(wildcard $(LIBZIP_SRC_PATH)/*.c $(LIBZIP_SRC_PATH)/zlib/*.c))))
+ LIBSPLASHSCREEN_STATIC_LIB_EXCLUDE_OBJS += $(LIBZIP_OBJS)
endif
LIBSPLASHSCREEN_CFLAGS += -DSPLASHSCREEN -DPNG_NO_MMX_CODE \
@@ -207,6 +214,8 @@ ifeq ($(ENABLE_HEADLESS_ONLY), false)
LIBSPLASHSCREEN_CFLAGS += -DWITH_X11 $(X_CFLAGS)
endif
+ LIBSPLASHSCREEN_STATIC_LIB_EXCLUDE_OBJS += systemScale.o
+
$(eval $(call SetupJdkLibrary, BUILD_LIBSPLASHSCREEN, \
NAME := splashscreen, \
EXTRA_SRC := $(LIBSPLASHSCREEN_EXTRA_SRC), \
@@ -257,6 +266,7 @@ ifeq ($(ENABLE_HEADLESS_ONLY), false)
-framework Security, \
LIBS_aix := $(LIBDL) -liconv $(X_LIBS) -lX11 -lXext, \
LIBS_windows := delayimp.lib gdi32.lib kernel32.lib user32.lib, \
+ STATIC_LIB_EXCLUDE_OBJS := $(LIBSPLASHSCREEN_STATIC_LIB_EXCLUDE_OBJS), \
))
TARGETS += $(BUILD_LIBSPLASHSCREEN)
diff --git a/make/modules/java.xml/Java.gmk b/make/modules/java.xml/Java.gmk
index 22c1dde2c2b..0c174f2113e 100644
--- a/make/modules/java.xml/Java.gmk
+++ b/make/modules/java.xml/Java.gmk
@@ -27,5 +27,5 @@ DISABLED_WARNINGS_java += dangling-doc-comments lossy-conversions this-escape
DOCLINT += -Xdoclint:all/protected \
'-Xdoclint/package:$(call CommaList, javax.xml.catalog javax.xml.datatype \
javax.xml.transform javax.xml.validation javax.xml.xpath)'
-COPY += .dtd .xsd .xml
+COPY += .dtd .xsd .xml .ent .mod
CLEAN += .properties
diff --git a/make/modules/jdk.accessibility/Lib.gmk b/make/modules/jdk.accessibility/Lib.gmk
index 6a429a56375..6323049c985 100644
--- a/make/modules/jdk.accessibility/Lib.gmk
+++ b/make/modules/jdk.accessibility/Lib.gmk
@@ -38,6 +38,7 @@ ifeq ($(call isTargetOs, windows), true)
NAME := javaaccessbridge, \
EXTRA_SRC := common, \
OPTIMIZATION := LOW, \
+ ONLY_EXPORTED := true, \
DISABLED_WARNINGS_microsoft := 4311 4302 4312, \
CXXFLAGS_FILTER_OUT := -MD, \
CXXFLAGS := -MT -DACCESSBRIDGE_ARCH_64, \
@@ -67,6 +68,7 @@ ifeq ($(call isTargetOs, windows), true)
CXXFLAGS := -DACCESSBRIDGE_ARCH_64, \
EXTRA_HEADER_DIRS := \
include/bridge, \
+ ONLY_EXPORTED := true, \
LDFLAGS := \
-def:$(ACCESSIBILITY_SRCDIR)/libwindowsaccessbridge/WinAccessBridge.DEF, \
LIBS_windows := advapi32.lib comdlg32.lib gdi32.lib kernel32.lib \
diff --git a/make/modules/jdk.jpackage/Lib.gmk b/make/modules/jdk.jpackage/Lib.gmk
index 75548133019..33d10336e6e 100644
--- a/make/modules/jdk.jpackage/Lib.gmk
+++ b/make/modules/jdk.jpackage/Lib.gmk
@@ -49,6 +49,7 @@ $(eval $(call SetupJdkExecutable, BUILD_JPACKAGEAPPLAUNCHER, \
LINK_TYPE := $(JPACKAGEAPPLAUNCHER_LINK_TYPE), \
OUTPUT_DIR := $(JPACKAGE_OUTPUT_DIR), \
SYMBOLS_DIR := $(SUPPORT_OUTPUTDIR)/native/$(MODULE)/jpackageapplauncher, \
+ ONLY_EXPORTED := true, \
SRC := applauncher, \
EXTRA_SRC := common, \
INCLUDE_FILES := $(JPACKAGEAPPLAUNCHER_INCLUDE_FILES), \
@@ -83,6 +84,7 @@ ifeq ($(call isTargetOs, linux), true)
OUTPUT_DIR := $(JPACKAGE_OUTPUT_DIR), \
SYMBOLS_DIR := \
$(SUPPORT_OUTPUTDIR)/native/$(MODULE)/libjpackageapplauncheraux, \
+ ONLY_EXPORTED := true, \
SRC := libapplauncher, \
EXTRA_SRC := \
applauncher \
@@ -127,6 +129,7 @@ ifeq ($(call isTargetOs, windows), true)
NAME := wixhelper, \
OUTPUT_DIR := $(JPACKAGE_OUTPUT_DIR), \
SYMBOLS_DIR := $(SUPPORT_OUTPUTDIR)/native/$(MODULE)/libwixhelper, \
+ ONLY_EXPORTED := true, \
OPTIMIZATION := LOW, \
EXTRA_SRC := common, \
CXXFLAGS_FILTER_OUT := -MD, \
@@ -146,6 +149,7 @@ ifeq ($(call isTargetOs, windows), true)
NAME := msiwrapper, \
OUTPUT_DIR := $(JPACKAGE_OUTPUT_DIR), \
SYMBOLS_DIR := $(SUPPORT_OUTPUTDIR)/native/$(MODULE)/msiwrapper, \
+ ONLY_EXPORTED := true, \
EXTRA_SRC := common, \
CXXFLAGS_FILTER_OUT := -MD, \
CXXFLAGS_windows := -MT $(JPACKAGE_CXXFLAGS_windows), \
@@ -164,6 +168,7 @@ ifeq ($(call isTargetOs, windows), true)
OUTPUT_DIR := $(JPACKAGE_OUTPUT_DIR), \
SYMBOLS_DIR := \
$(SUPPORT_OUTPUTDIR)/native/$(MODULE)/jpackageapplauncherw, \
+ ONLY_EXPORTED := true, \
SRC := applauncher, \
EXTRA_SRC := common, \
OPTIMIZATION := LOW, \
diff --git a/make/test/BuildMicrobenchmark.gmk b/make/test/BuildMicrobenchmark.gmk
index 0a4c2d9a48d..da9ff2029e8 100644
--- a/make/test/BuildMicrobenchmark.gmk
+++ b/make/test/BuildMicrobenchmark.gmk
@@ -89,6 +89,7 @@ $(eval $(call SetupJavaCompilation, BUILD_JDK_MICROBENCHMARK, \
SRC := $(MICROBENCHMARK_SRC), \
BIN := $(MICROBENCHMARK_CLASSES), \
JAVAC_FLAGS := \
+ --add-exports java.base/jdk.internal.classfile.components=ALL-UNNAMED \
--add-exports java.base/jdk.internal.classfile.impl=ALL-UNNAMED \
--add-exports java.base/jdk.internal.event=ALL-UNNAMED \
--add-exports java.base/jdk.internal.foreign=ALL-UNNAMED \
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
index 3b0c8ae432c..5cc57976296 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
@@ -206,8 +206,8 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
// Handle existing monitor.
bind(object_has_monitor);
- // Try to CAS owner (no owner => current thread's _lock_id).
- ldr(rscratch2, Address(rthread, JavaThread::lock_id_offset()));
+ // Try to CAS owner (no owner => current thread's _monitor_owner_id).
+ ldr(rscratch2, Address(rthread, JavaThread::monitor_owner_id_offset()));
add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value));
cmpxchg(tmp, zr, rscratch2, Assembler::xword, /*acquire*/ true,
/*release*/ true, /*weak*/ false, tmp3Reg); // Sets flags for result
@@ -469,8 +469,8 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Regist
// Compute owner address.
lea(t2_owner_addr, owner_address);
- // Try to CAS owner (no owner => current thread's _lock_id).
- ldr(rscratch2, Address(rthread, JavaThread::lock_id_offset()));
+ // Try to CAS owner (no owner => current thread's _monitor_owner_id).
+ ldr(rscratch2, Address(rthread, JavaThread::monitor_owner_id_offset()));
cmpxchg(t2_owner_addr, zr, rscratch2, Assembler::xword, /*acquire*/ true,
/*release*/ false, /*weak*/ false, t3_owner);
br(Assembler::EQ, monitor_locked);
diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp
index 261502d5823..666335330ed 100644
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -86,6 +87,10 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LI
LIR_Opr result = gen->new_register(T_INT);
__ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result));
+
+ if (ShenandoahCardBarrier) {
+ post_barrier(access, access.resolved_addr(), new_value.result());
+ }
return result;
}
}
@@ -113,6 +118,9 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt
pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
result /* pre_val */);
}
+ if (ShenandoahCardBarrier) {
+ post_barrier(access, access.resolved_addr(), result);
+ }
}
return result;
diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
index 84d06dbcc7b..f682e86cdfb 100644
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -31,6 +32,7 @@
#include "gc/shenandoah/shenandoahRuntime.hpp"
#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/mode/shenandoahMode.hpp"
#include "interpreter/interpreter.hpp"
#include "interpreter/interp_masm.hpp"
#include "runtime/javaThread.hpp"
@@ -77,6 +79,13 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
}
}
+void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+ Register start, Register count, Register tmp, RegSet saved_regs) {
+ if (ShenandoahCardBarrier && is_oop) {
+ gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp, saved_regs);
+ }
+}
+
void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
Register obj,
Register pre_val,
@@ -362,6 +371,26 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
}
}
+void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
+ assert(ShenandoahCardBarrier, "Should have been checked by caller");
+
+ __ lsr(obj, obj, CardTable::card_shift());
+
+ assert(CardTable::dirty_card_val() == 0, "must be");
+
+ __ load_byte_map_base(rscratch1);
+
+ if (UseCondCardMark) {
+ Label L_already_dirty;
+ __ ldrb(rscratch2, Address(obj, rscratch1));
+ __ cbz(rscratch2, L_already_dirty);
+ __ strb(zr, Address(obj, rscratch1));
+ __ bind(L_already_dirty);
+ } else {
+ __ strb(zr, Address(obj, rscratch1));
+ }
+}
+
void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
bool on_oop = is_reference_type(type);
@@ -387,18 +416,13 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet
val != noreg /* tosca_live */,
false /* expand_call */);
- if (val == noreg) {
- BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg);
- } else {
- // Barrier needs uncompressed oop for region cross check.
- Register new_val = val;
- if (UseCompressedOops) {
- new_val = rscratch2;
- __ mov(new_val, val);
- }
- BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
- }
+ BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
+ bool in_heap = (decorators & IN_HEAP) != 0;
+ bool needs_post_barrier = (val != noreg) && in_heap && ShenandoahCardBarrier;
+ if (needs_post_barrier) {
+ store_check(masm, tmp3);
+ }
}
void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
@@ -581,6 +605,35 @@ void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
}
}
+void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+ Register start, Register count, Register scratch, RegSet saved_regs) {
+ assert(ShenandoahCardBarrier, "Should have been checked by caller");
+
+ Label L_loop, L_done;
+ const Register end = count;
+
+ // Zero count? Nothing to do.
+ __ cbz(count, L_done);
+
+ // end = start + count << LogBytesPerHeapOop
+ // last element address to make inclusive
+ __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
+ __ sub(end, end, BytesPerHeapOop);
+ __ lsr(start, start, CardTable::card_shift());
+ __ lsr(end, end, CardTable::card_shift());
+
+ // number of bytes to copy
+ __ sub(count, end, start);
+
+ __ load_byte_map_base(scratch);
+ __ add(start, start, scratch);
+ __ bind(L_loop);
+ __ strb(zr, Address(start, count));
+ __ subs(count, count, 1);
+ __ br(Assembler::GE, L_loop);
+ __ bind(L_done);
+}
+
#undef __
#ifdef COMPILER1
diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp
index ee11b2e73f7..a12d4e2beec 100644
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -55,10 +56,16 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
bool tosca_live,
bool expand_call);
+ void store_check(MacroAssembler* masm, Register obj);
+
void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg);
void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg);
void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators);
+ void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+ Register start, Register count,
+ Register scratch, RegSet saved_regs);
+
public:
virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_data_patch; }
@@ -71,6 +78,8 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
Register src, Register dst, Register count, RegSet saved_regs);
+ virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+ Register start, Register count, Register tmp, RegSet saved_regs);
virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp2);
virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
index ce71d3a2262..a836d71205e 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
@@ -5305,7 +5305,7 @@ MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode() {
if (operand_valid_for_logical_immediate(
/*is32*/false, (uint64_t)CompressedKlassPointers::base())) {
const size_t range = CompressedKlassPointers::klass_range_end() - CompressedKlassPointers::base();
- const uint64_t range_mask = (1ULL << log2i(range)) - 1;
+ const uint64_t range_mask = right_n_bits(log2i_ceil(range));
if (((uint64_t)CompressedKlassPointers::base() & range_mask) == 0) {
return (_klass_decode_mode = KlassDecodeXor);
}
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
index cf16fe3a08d..87c7862e250 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
@@ -444,7 +444,19 @@ void VM_Version::initialize() {
}
if (UseSVE > 0) {
- _initial_sve_vector_length = get_current_sve_vector_length();
+ int vl = get_current_sve_vector_length();
+ if (vl < 0) {
+ warning("Unable to get SVE vector length on this system. "
+ "Disabling SVE. Specify -XX:UseSVE=0 to shun this warning.");
+ FLAG_SET_DEFAULT(UseSVE, 0);
+ } else if ((vl == 0) || ((vl % FloatRegister::sve_vl_min) != 0) || !is_power_of_2(vl)) {
+ warning("Detected SVE vector length (%d) should be a power of two and a multiple of %d. "
+ "Disabling SVE. Specify -XX:UseSVE=0 to shun this warning.",
+ vl, FloatRegister::sve_vl_min);
+ FLAG_SET_DEFAULT(UseSVE, 0);
+ } else {
+ _initial_sve_vector_length = vl;
+ }
}
// This machine allows unaligned memory accesses
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.hpp b/src/hotspot/cpu/ppc/assembler_ppc.hpp
index d2584cc1af3..b99235371aa 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.hpp
@@ -506,6 +506,9 @@ class Assembler : public AbstractAssembler {
LFSU_OPCODE = (49u << OPCODE_SHIFT | 00u << 1),
LFSX_OPCODE = (31u << OPCODE_SHIFT | 535u << 1),
+ LFIWAX_OPCODE = (31u << OPCODE_SHIFT | 855u << 1),
+ LFIWZX_OPCODE = (31u << OPCODE_SHIFT | 887u << 1),
+
STFD_OPCODE = (54u << OPCODE_SHIFT | 00u << 1),
STFDU_OPCODE = (55u << OPCODE_SHIFT | 00u << 1),
STFDX_OPCODE = (31u << OPCODE_SHIFT | 727u << 1),
@@ -513,6 +516,8 @@ class Assembler : public AbstractAssembler {
STFSU_OPCODE = (53u << OPCODE_SHIFT | 00u << 1),
STFSX_OPCODE = (31u << OPCODE_SHIFT | 663u << 1),
+ STFIWX_OPCODE = (31u << OPCODE_SHIFT | 983u << 1),
+
FSQRT_OPCODE = (63u << OPCODE_SHIFT | 22u << 1), // A-FORM
FSQRTS_OPCODE = (59u << OPCODE_SHIFT | 22u << 1), // A-FORM
@@ -555,6 +560,10 @@ class Assembler : public AbstractAssembler {
XVDIVSP_OPCODE = (60u << OPCODE_SHIFT | 88u << 3),
XXBRD_OPCODE = (60u << OPCODE_SHIFT | 475u << 2 | 23u << 16), // XX2-FORM
XXBRW_OPCODE = (60u << OPCODE_SHIFT | 475u << 2 | 15u << 16), // XX2-FORM
+ XVCVHPSP_OPCODE= (60u << OPCODE_SHIFT | 475u << 2 | 24u << 16), // XX2-FORM
+ XVCVSPHP_OPCODE= (60u << OPCODE_SHIFT | 475u << 2 | 25u << 16), // XX2-FORM
+ XSCVHPDP_OPCODE= (60u << OPCODE_SHIFT | 347u << 2 | 16u << 16), // XX2-FORM
+ XSCVDPHP_OPCODE= (60u << OPCODE_SHIFT | 347u << 2 | 17u << 16), // XX2-FORM
XXPERM_OPCODE = (60u << OPCODE_SHIFT | 26u << 3),
XXSEL_OPCODE = (60u << OPCODE_SHIFT | 3u << 4),
XXSPLTIB_OPCODE= (60u << OPCODE_SHIFT | 360u << 1),
@@ -2076,6 +2085,9 @@ class Assembler : public AbstractAssembler {
inline void lfdu( FloatRegister d, int si16, Register a);
inline void lfdx( FloatRegister d, Register a, Register b);
+ inline void lfiwax(FloatRegister d, Register a, Register b);
+ inline void lfiwzx(FloatRegister d, Register a, Register b);
+
// PPC 1, section 4.6.3 Floating-Point Store Instructions
inline void stfs( FloatRegister s, int si16, Register a);
inline void stfsu( FloatRegister s, int si16, Register a);
@@ -2084,6 +2096,8 @@ class Assembler : public AbstractAssembler {
inline void stfdu( FloatRegister s, int si16, Register a);
inline void stfdx( FloatRegister s, Register a, Register b);
+ inline void stfiwx(FloatRegister s, Register a, Register b);
+
// PPC 1, section 4.6.4 Floating-Point Move Instructions
inline void fmr( FloatRegister d, FloatRegister b);
inline void fmr_( FloatRegister d, FloatRegister b);
@@ -2348,6 +2362,10 @@ class Assembler : public AbstractAssembler {
inline void xxleqv( VectorSRegister d, VectorSRegister a, VectorSRegister b);
inline void xxbrd( VectorSRegister d, VectorSRegister b);
inline void xxbrw( VectorSRegister d, VectorSRegister b);
+ inline void xvcvhpsp( VectorSRegister d, VectorSRegister b);
+ inline void xvcvsphp( VectorSRegister d, VectorSRegister b);
+ inline void xscvhpdp( VectorSRegister d, VectorSRegister b);
+ inline void xscvdphp( VectorSRegister d, VectorSRegister b);
inline void xxland( VectorSRegister d, VectorSRegister a, VectorSRegister b);
inline void xxsel( VectorSRegister d, VectorSRegister a, VectorSRegister b, VectorSRegister c);
inline void xxspltib( VectorSRegister d, int ui8);
@@ -2474,10 +2492,13 @@ class Assembler : public AbstractAssembler {
inline void lfsx( FloatRegister d, Register b);
inline void lfd( FloatRegister d, int si16);
inline void lfdx( FloatRegister d, Register b);
+ inline void lfiwax(FloatRegister d, Register b);
+ inline void lfiwzx(FloatRegister d, Register b);
inline void stfs( FloatRegister s, int si16);
inline void stfsx( FloatRegister s, Register b);
inline void stfd( FloatRegister s, int si16);
inline void stfdx( FloatRegister s, Register b);
+ inline void stfiwx(FloatRegister s, Register b);
inline void lvebx( VectorRegister d, Register s2);
inline void lvehx( VectorRegister d, Register s2);
inline void lvewx( VectorRegister d, Register s2);
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
index 07de57babfa..eae75da9fbf 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
@@ -741,6 +741,9 @@ inline void Assembler::lfd( FloatRegister d, int si16, Register a) { emit_int3
inline void Assembler::lfdu(FloatRegister d, int si16, Register a) { emit_int32( LFDU_OPCODE | frt(d) | ra(a) | simm(si16,16)); }
inline void Assembler::lfdx(FloatRegister d, Register a, Register b) { emit_int32( LFDX_OPCODE | frt(d) | ra0mem(a) | rb(b)); }
+inline void Assembler::lfiwax(FloatRegister d, Register a, Register b) { emit_int32( LFIWAX_OPCODE | frt(d) | ra0mem(a) |rb(b)); }
+inline void Assembler::lfiwzx(FloatRegister d, Register a, Register b) { emit_int32( LFIWZX_OPCODE | frt(d) | ra0mem(a) |rb(b)); }
+
// PPC 1, section 4.6.3 Floating-Point Store Instructions
// Use ra0mem instead of ra in some instructions below.
inline void Assembler::stfs( FloatRegister s, int si16, Register a) { emit_int32( STFS_OPCODE | frs(s) | ra0mem(a) | simm(si16,16)); }
@@ -750,6 +753,8 @@ inline void Assembler::stfd( FloatRegister s, int si16, Register a) { emit_int3
inline void Assembler::stfdu(FloatRegister s, int si16, Register a) { emit_int32( STFDU_OPCODE | frs(s) | ra(a) | simm(si16,16)); }
inline void Assembler::stfdx(FloatRegister s, Register a, Register b){ emit_int32( STFDX_OPCODE | frs(s) | ra0mem(a) | rb(b)); }
+inline void Assembler::stfiwx(FloatRegister s, Register a, Register b) { emit_int32( STFIWX_OPCODE | frs(s) | ra0mem(a) |rb(b)); }
+
// PPC 1, section 4.6.4 Floating-Point Move Instructions
inline void Assembler::fmr( FloatRegister d, FloatRegister b) { emit_int32( FMR_OPCODE | frt(d) | frb(b) | rc(0)); }
inline void Assembler::fmr_(FloatRegister d, FloatRegister b) { emit_int32( FMR_OPCODE | frt(d) | frb(b) | rc(1)); }
@@ -871,6 +876,10 @@ inline void Assembler::xxlxor( VectorSRegister d, VectorSRegister a, VectorSReg
inline void Assembler::xxleqv( VectorSRegister d, VectorSRegister a, VectorSRegister b) { emit_int32( XXLEQV_OPCODE | vsrt(d) | vsra(a) | vsrb(b)); }
inline void Assembler::xxbrd( VectorSRegister d, VectorSRegister b) { emit_int32( XXBRD_OPCODE | vsrt(d) | vsrb(b) ); }
inline void Assembler::xxbrw( VectorSRegister d, VectorSRegister b) { emit_int32( XXBRW_OPCODE | vsrt(d) | vsrb(b) ); }
+inline void Assembler::xvcvhpsp(VectorSRegister d, VectorSRegister b) { emit_int32( XVCVHPSP_OPCODE | vsrt(d) | vsrb(b) ); }
+inline void Assembler::xvcvsphp(VectorSRegister d, VectorSRegister b) { emit_int32( XVCVSPHP_OPCODE | vsrt(d) | vsrb(b) ); }
+inline void Assembler::xscvhpdp(VectorSRegister d, VectorSRegister b) { emit_int32( XSCVHPDP_OPCODE | vsrt(d) | vsrb(b) ); }
+inline void Assembler::xscvdphp(VectorSRegister d, VectorSRegister b) { emit_int32( XSCVDPHP_OPCODE | vsrt(d) | vsrb(b) ); }
inline void Assembler::xvdivsp( VectorSRegister d, VectorSRegister a, VectorSRegister b) { emit_int32( XVDIVSP_OPCODE | vsrt(d) | vsra(a) | vsrb(b)); }
inline void Assembler::xvdivdp( VectorSRegister d, VectorSRegister a, VectorSRegister b) { emit_int32( XVDIVDP_OPCODE | vsrt(d) | vsra(a) | vsrb(b)); }
inline void Assembler::xvabssp( VectorSRegister d, VectorSRegister b) { emit_int32( XVABSSP_OPCODE | vsrt(d) | vsrb(b)); }
@@ -1150,12 +1159,17 @@ inline void Assembler::lfsx(FloatRegister d, Register b) { emit_int32( LFSX_OPCO
inline void Assembler::lfd( FloatRegister d, int si16) { emit_int32( LFD_OPCODE | frt(d) | simm(si16,16)); }
inline void Assembler::lfdx(FloatRegister d, Register b) { emit_int32( LFDX_OPCODE | frt(d) | rb(b)); }
+inline void Assembler::lfiwax(FloatRegister d, Register b) { emit_int32( LFIWAX_OPCODE | frt(d) | rb(b)); }
+inline void Assembler::lfiwzx(FloatRegister d, Register b) { emit_int32( LFIWZX_OPCODE | frt(d) | rb(b)); }
+
// ra0 version
inline void Assembler::stfs( FloatRegister s, int si16) { emit_int32( STFS_OPCODE | frs(s) | simm(si16, 16)); }
inline void Assembler::stfsx(FloatRegister s, Register b) { emit_int32( STFSX_OPCODE | frs(s) | rb(b)); }
inline void Assembler::stfd( FloatRegister s, int si16) { emit_int32( STFD_OPCODE | frs(s) | simm(si16, 16)); }
inline void Assembler::stfdx(FloatRegister s, Register b) { emit_int32( STFDX_OPCODE | frs(s) | rb(b)); }
+inline void Assembler::stfiwx(FloatRegister s, Register b) { emit_int32( STFIWX_OPCODE | frs(s) |rb(b)); }
+
// ra0 version
inline void Assembler::lvebx( VectorRegister d, Register s2) { emit_int32( LVEBX_OPCODE | vrt(d) | rb(s2)); }
inline void Assembler::lvehx( VectorRegister d, Register s2) { emit_int32( LVEHX_OPCODE | vrt(d) | rb(s2)); }
diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
index 9002ac2042d..a22e93aa1c7 100644
--- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
@@ -1718,7 +1718,7 @@ void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr
}
-void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr thread, LIR_Opr dest, LIR_Op* op) {
+void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
switch (code) {
case lir_sqrt: {
__ fsqrt(dest->as_double_reg(), value->as_double_reg());
@@ -1728,6 +1728,14 @@ void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr thread, L
__ fabs(dest->as_double_reg(), value->as_double_reg());
break;
}
+ case lir_f2hf: {
+ __ f2hf(dest.as_register(), value.as_float_reg(), tmp.as_float_reg());
+ break;
+ }
+ case lir_hf2f: {
+ __ hf2f(dest->as_float_reg(), value.as_register());
+ break;
+ }
default: {
ShouldNotReachHere();
break;
diff --git a/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp
index 7973e9d0545..4e0908d6e61 100644
--- a/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp
@@ -690,6 +690,25 @@ void LIRGenerator::do_MathIntrinsic(Intrinsic* x) {
__ abs(value.result(), dst, LIR_OprFact::illegalOpr);
break;
}
+ case vmIntrinsics::_floatToFloat16: {
+ assert(x->number_of_arguments() == 1, "wrong type");
+ LIRItem value(x->argument_at(0), this);
+ value.load_item();
+ LIR_Opr dst = rlock_result(x);
+ LIR_Opr tmp = new_register(T_FLOAT);
+ // f2hf treats tmp as live_in. Workaround: initialize to some value.
+ __ move(LIR_OprFact::floatConst(-0.0), tmp); // just to satisfy LinearScan
+ __ f2hf(value.result(), dst, tmp);
+ break;
+ }
+ case vmIntrinsics::_float16ToFloat: {
+ assert(x->number_of_arguments() == 1, "wrong type");
+ LIRItem value(x->argument_at(0), this);
+ value.load_item();
+ LIR_Opr dst = rlock_result(x);
+ __ hf2f(value.result(), dst, LIR_OprFact::illegalOpr);
+ break;
+ }
case vmIntrinsics::_dsqrt:
case vmIntrinsics::_dsqrt_strict: {
if (VM_Version::has_fsqrt()) {
diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp
index 5f87281bdf9..ce12d1fcf03 100644
--- a/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp
@@ -102,6 +102,10 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess &access, LI
__ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result));
+ if (ShenandoahCardBarrier) {
+ post_barrier(access, access.resolved_addr(), new_value.result());
+ }
+
return result;
}
}
@@ -132,6 +136,10 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess &access, LIRIt
if (ShenandoahSATBBarrier) {
pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr, result);
}
+
+ if (ShenandoahCardBarrier) {
+ post_barrier(access, access.resolved_addr(), result);
+ }
}
return result;
diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
index 53150807212..796c32a58df 100644
--- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
@@ -36,6 +36,7 @@
#include "gc/shenandoah/shenandoahRuntime.hpp"
#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/mode/shenandoahMode.hpp"
#include "interpreter/interpreter.hpp"
#include "macroAssembler_ppc.hpp"
#include "runtime/javaThread.hpp"
@@ -76,8 +77,6 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm,
void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
Register src, Register dst, Register count,
Register preserve1, Register preserve2) {
- __ block_comment("arraycopy_prologue (shenandoahgc) {");
-
Register R11_tmp = R11_scratch1;
assert_different_registers(src, dst, count, R11_tmp, noreg);
@@ -100,6 +99,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, Dec
return;
}
+ __ block_comment("arraycopy_prologue (shenandoahgc) {");
Label skip_prologue;
// Fast path: Array is of length zero.
@@ -173,6 +173,16 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, Dec
__ block_comment("} arraycopy_prologue (shenandoahgc)");
}
+void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+ Register dst, Register count,
+ Register preserve) {
+ if (ShenandoahCardBarrier && is_reference_type(type)) {
+ __ block_comment("arraycopy_epilogue (shenandoahgc) {");
+ gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve);
+ __ block_comment("} arraycopy_epilogue (shenandoahgc)");
+ }
+}
+
// The to-be-enqueued value can either be determined
// - dynamically by passing the reference's address information (load mode) or
// - statically by passing a register the value is stored in (preloaded mode)
@@ -576,6 +586,25 @@ void ShenandoahBarrierSetAssembler::load_at(
}
}
+void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
+ assert(ShenandoahCardBarrier, "Should have been checked by caller");
+
+ ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
+ CardTable* ct = ctbs->card_table();
+ assert_different_registers(base, tmp, R0);
+
+ if (ind_or_offs.is_constant()) {
+ __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp);
+ } else {
+ __ add(base, ind_or_offs.as_register(), base);
+ }
+
+ __ load_const_optimized(tmp, (address)ct->byte_map_base(), R0);
+ __ srdi(base, base, CardTable::card_shift());
+ __ li(R0, CardTable::dirty_card_val());
+ __ stbx(R0, tmp, base);
+}
+
// base: Base register of the reference's address.
// ind_or_offs: Index or offset of the reference's address.
// val: To-be-stored value/reference's new value.
@@ -594,6 +623,11 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet
val,
tmp1, tmp2, tmp3,
preservation_level);
+
+ // No need for post barrier if storing NULL
+ if (ShenandoahCardBarrier && is_reference_type(type) && val != noreg) {
+ store_check(masm, base, ind_or_offs, tmp1);
+ }
}
void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
@@ -743,6 +777,40 @@ void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler *masm, Register b
__ block_comment("} cmpxchg_oop (shenandoahgc)");
}
+void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+ Register addr, Register count, Register preserve) {
+ assert(ShenandoahCardBarrier, "Should have been checked by caller");
+
+ ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
+ CardTable* ct = bs->card_table();
+ assert_different_registers(addr, count, R0);
+
+ Label L_skip_loop, L_store_loop;
+
+ __ sldi_(count, count, LogBytesPerHeapOop);
+
+ // Zero length? Skip.
+ __ beq(CCR0, L_skip_loop);
+
+ __ addi(count, count, -BytesPerHeapOop);
+ __ add(count, addr, count);
+ // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
+ __ srdi(addr, addr, CardTable::card_shift());
+ __ srdi(count, count, CardTable::card_shift());
+ __ subf(count, addr, count);
+ __ add_const_optimized(addr, addr, (address)ct->byte_map_base(), R0);
+ __ addi(count, count, 1);
+ __ li(R0, 0);
+ __ mtctr(count);
+
+ // Byte store loop
+ __ bind(L_store_loop);
+ __ stb(R0, 0, addr);
+ __ addi(addr, addr, 1);
+ __ bdnz(L_store_loop);
+ __ bind(L_skip_loop);
+}
+
#undef __
#ifdef COMPILER1
diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp
index 2e56187c169..6ee70b4b4ea 100644
--- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp
@@ -51,6 +51,10 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
Register tmp1, Register tmp2,
MacroAssembler::PreservationLevel preservation_level);
+ void store_check(MacroAssembler* masm,
+ Register base, RegisterOrConstant ind_or_offs,
+ Register tmp);
+
void load_reference_barrier_impl(MacroAssembler* masm, DecoratorSet decorators,
Register base, RegisterOrConstant ind_or_offs,
Register dst,
@@ -60,6 +64,10 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
/* ==== Helper methods for barrier implementations ==== */
void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp);
+ void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+ Register addr, Register count,
+ Register preserve);
+
public:
virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_data_patch; }
@@ -95,7 +103,11 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
/* ==== Access api ==== */
virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
- Register src, Register dst, Register count, Register preserve1, Register preserve2);
+ Register src, Register dst, Register count,
+ Register preserve1, Register preserve2);
+ virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+ Register dst, Register count,
+ Register preserve);
virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register base, RegisterOrConstant ind_or_offs, Register val,
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
index 994521ba48f..511f0f2e96c 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
@@ -2664,10 +2664,10 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
// Handle existing monitor.
bind(object_has_monitor);
- // Try to CAS owner (no owner => current thread's _lock_id).
+ // Try to CAS owner (no owner => current thread's _monitor_owner_id).
addi(temp, displaced_header, in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value);
Register thread_id = displaced_header;
- ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
+ ld(thread_id, in_bytes(JavaThread::monitor_owner_id_offset()), R16_thread);
cmpxchgd(/*flag=*/flag,
/*current_value=*/current_header,
/*compare_value=*/(intptr_t)0,
@@ -2944,9 +2944,9 @@ void MacroAssembler::compiler_fast_lock_lightweight_object(ConditionRegister fla
addi(owner_addr, monitor, in_bytes(ObjectMonitor::owner_offset()));
}
- // Try to CAS owner (no owner => current thread's _lock_id).
+ // Try to CAS owner (no owner => current thread's _monitor_owner_id).
assert_different_registers(thread_id, monitor, owner_addr, box, R0);
- ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
+ ld(thread_id, in_bytes(JavaThread::monitor_owner_id_offset()), R16_thread);
cmpxchgd(/*flag=*/CCR0,
/*current_value=*/R0,
/*compare_value=*/(intptr_t)0,
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
index 237024fa05e..3e82c1c6785 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
@@ -186,6 +186,9 @@ class MacroAssembler: public Assembler {
void inline set_cmpu3(Register dst, bool treat_unordered_like_less);
// Branch-free implementation to convert !=0 to 1.
void inline normalize_bool(Register dst, Register temp = R0, bool is_64bit = false);
+ // Convert between half precision float encoded into a short and a float in a FloatRegister.
+ void inline f2hf(Register dst, FloatRegister src, FloatRegister tmp);
+ void inline hf2f(FloatRegister dst, Register src);
inline void pd_patch_instruction(address branch, address target, const char* file, int line);
NOT_PRODUCT(static void pd_print_patched_instruction(address branch);)
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp
index f98ad592388..6f5cd8fbd96 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp
@@ -297,6 +297,20 @@ inline void MacroAssembler::normalize_bool(Register dst, Register temp, bool is_
}
}
+inline void MacroAssembler::f2hf(Register dst, FloatRegister src, FloatRegister tmp) {
+ // Single precision values in FloatRegisters use double precision format on PPC64.
+ xscvdphp(tmp->to_vsr(), src->to_vsr());
+ mffprd(dst, tmp);
+ // Make it a proper short (sign-extended).
+ extsh(dst, dst);
+}
+
+inline void MacroAssembler::hf2f(FloatRegister dst, Register src) {
+ mtfprd(dst, src);
+ // Single precision values in FloatRegisters use double precision format on PPC64.
+ xscvhpdp(dst->to_vsr(), dst->to_vsr());
+}
+
// Convenience bc_far versions
inline void MacroAssembler::blt_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs1, bi0(crx, less), L, optimize); }
inline void MacroAssembler::bgt_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs1, bi0(crx, greater), L, optimize); }
diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad
index cfed8674e89..549ae0740e2 100644
--- a/src/hotspot/cpu/ppc/ppc.ad
+++ b/src/hotspot/cpu/ppc/ppc.ad
@@ -2077,6 +2077,9 @@ bool Matcher::match_rule_supported(int opcode) {
case Op_PopCountI:
case Op_PopCountL:
return (UsePopCountInstruction && VM_Version::has_popcntw());
+ case Op_ConvF2HF:
+ case Op_ConvHF2F:
+ return VM_Version::supports_float16();
case Op_AddVB:
case Op_AddVS:
@@ -11245,6 +11248,34 @@ instruct convF2D_reg(regD dst, regF src) %{
ins_pipe(pipe_class_default);
%}
+instruct convF2HF_reg_reg(iRegIdst dst, regF src, regF tmp) %{
+ match(Set dst (ConvF2HF src));
+ effect(TEMP tmp);
+ ins_cost(3 * DEFAULT_COST);
+ size(12);
+ format %{ "xscvdphp $tmp, $src\t# convert to half precision\n\t"
+ "mffprd $dst, $tmp\t# move result from $tmp to $dst\n\t"
+ "extsh $dst, $dst\t# make it a proper short"
+ %}
+ ins_encode %{
+ __ f2hf($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct convHF2F_reg_reg(regF dst, iRegIsrc src) %{
+ match(Set dst (ConvHF2F src));
+ ins_cost(2 * DEFAULT_COST);
+ size(8);
+ format %{ "mtfprd $dst, $src\t# move source from $src to $dst\n\t"
+ "xscvhpdp $dst, $dst\t# convert from half precision"
+ %}
+ ins_encode %{
+ __ hf2f($dst$$FloatRegister, $src$$Register);
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
//----------Control Flow Instructions------------------------------------------
// Compare Instructions
diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
index 9f2b668b9c8..00d2e206bf9 100644
--- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
+++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
@@ -3451,6 +3451,24 @@ class StubGenerator: public StubCodeGenerator {
return start;
}
+ address generate_floatToFloat16() {
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, "StubRoutines", "floatToFloat16");
+ address start = __ function_entry();
+ __ f2hf(R3_RET, F1_ARG1, F0);
+ __ blr();
+ return start;
+ }
+
+ address generate_float16ToFloat() {
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, "StubRoutines", "float16ToFloat");
+ address start = __ function_entry();
+ __ hf2f(F1_RET, R3_ARG1);
+ __ blr();
+ return start;
+ }
+
address generate_method_entry_barrier() {
__ align(CodeEntryAlignment);
StubCodeMark mark(this, "StubRoutines", "nmethod_entry_barrier");
@@ -4678,6 +4696,12 @@ address generate_lookup_secondary_supers_table_stub(u1 super_klass_index) {
StubRoutines::_crc32c_table_addr = StubRoutines::ppc::generate_crc_constants(REVERSE_CRC32C_POLY);
StubRoutines::_updateBytesCRC32C = generate_CRC32_updateBytes(true);
}
+
+ if (VM_Version::supports_float16()) {
+ // For results consistency both intrinsics should be enabled.
+ StubRoutines::_hf2f = generate_float16ToFloat();
+ StubRoutines::_f2hf = generate_floatToFloat16();
+ }
}
void generate_continuation_stubs() {
diff --git a/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp b/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp
index e320349583d..9147dfc1677 100644
--- a/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp
+++ b/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp
@@ -1155,6 +1155,44 @@ address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::M
return entry;
}
+address TemplateInterpreterGenerator::generate_Float_floatToFloat16_entry() {
+ if (!VM_Version::supports_float16()) return nullptr;
+
+ address entry = __ pc();
+
+ __ lfs(F1, Interpreter::stackElementSize, R15_esp);
+ __ f2hf(R3_RET, F1, F0);
+
+ // Restore caller sp for c2i case (from compiled) and for resized sender frame (from interpreted).
+ __ resize_frame_absolute(R21_sender_SP, R11_scratch1, R0);
+ __ blr();
+
+ __ flush();
+
+ return entry;
+}
+
+address TemplateInterpreterGenerator::generate_Float_float16ToFloat_entry() {
+ if (!VM_Version::supports_float16()) return nullptr;
+
+ address entry = __ pc();
+
+ // Note: Could also use:
+ //__ li(R3, Interpreter::stackElementSize);
+ //__ lfiwax(F1_RET, R15_esp, R3); // short stored as 32 bit integer
+ //__ xscvhpdp(F1_RET->to_vsr(), F1_RET->to_vsr());
+ __ lwa(R3, Interpreter::stackElementSize, R15_esp);
+ __ hf2f(F1_RET, R3);
+
+ // Restore caller sp for c2i case (from compiled) and for resized sender frame (from interpreted).
+ __ resize_frame_absolute(R21_sender_SP, R11_scratch1, R0);
+ __ blr();
+
+ __ flush();
+
+ return entry;
+}
+
void TemplateInterpreterGenerator::bang_stack_shadow_pages(bool native_call) {
// Quick & dirty stack overflow checking: bang the stack & handle trap.
// Note that we do the banging after the frame is setup, since the exception
@@ -1965,8 +2003,6 @@ address TemplateInterpreterGenerator::generate_Float_intBitsToFloat_entry() { re
address TemplateInterpreterGenerator::generate_Float_floatToRawIntBits_entry() { return nullptr; }
address TemplateInterpreterGenerator::generate_Double_longBitsToDouble_entry() { return nullptr; }
address TemplateInterpreterGenerator::generate_Double_doubleToRawLongBits_entry() { return nullptr; }
-address TemplateInterpreterGenerator::generate_Float_float16ToFloat_entry() { return nullptr; }
-address TemplateInterpreterGenerator::generate_Float_floatToFloat16_entry() { return nullptr; }
// =============================================================================
// Exceptions
diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.hpp b/src/hotspot/cpu/ppc/vm_version_ppc.hpp
index 9d8e4b88ee2..25dd07c9668 100644
--- a/src/hotspot/cpu/ppc/vm_version_ppc.hpp
+++ b/src/hotspot/cpu/ppc/vm_version_ppc.hpp
@@ -97,6 +97,8 @@ class VM_Version: public Abstract_VM_Version {
constexpr static bool supports_recursive_lightweight_locking() { return true; }
constexpr static bool supports_secondary_supers_table() { return true; }
+ static bool supports_float16() { return PowerArchitecturePPC64 >= 9; }
+
static bool is_determine_features_test_running() { return _is_determine_features_test_running; }
// CPU instruction support
static bool has_fsqrt() { return (_features & fsqrt_m) != 0; }
diff --git a/src/hotspot/cpu/riscv/assembler_riscv.hpp b/src/hotspot/cpu/riscv/assembler_riscv.hpp
index 7334ec675e3..31713d7362a 100644
--- a/src/hotspot/cpu/riscv/assembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/assembler_riscv.hpp
@@ -3107,6 +3107,38 @@ enum Nf {
#undef INSN
+// -------------- Zicond Instruction Definitions --------------
+// Zicond conditional operations extension
+ private:
+ enum CZERO_OP : unsigned int {
+ CZERO_NEZ = 0b111,
+ CZERO_EQZ = 0b101
+ };
+
+ template
+ void czero(Register Rd, Register Rs1, Register Rs2) {
+ assert_cond(UseZicond);
+ uint32_t insn = 0;
+ patch ((address)&insn, 6, 0, 0b0110011); // bits: 7, name: 0x33, attr: ['OP']
+ patch_reg((address)&insn, 7, Rd); // bits: 5, name: 'rd'
+ patch ((address)&insn, 14, 12, OP_VALUE); // bits: 3, name: 0x7, attr: ['CZERO.NEZ'] / 0x5, attr: ['CZERO.EQZ']}
+ patch_reg((address)&insn, 15, Rs1); // bits: 5, name: 'rs1', attr: ['value']
+ patch_reg((address)&insn, 20, Rs2); // bits: 5, name: 'rs2', attr: ['condition']
+ patch ((address)&insn, 31, 25, 0b0000111); // bits: 7, name: 0x7, attr: ['CZERO']
+ emit_int32(insn);
+ }
+
+ public:
+ // Moves zero to a register rd, if the condition rs2 is equal to zero, otherwise moves rs1 to rd.
+ void czero_eqz(Register rd, Register rs1_value, Register rs2_condition) {
+ czero(rd, rs1_value, rs2_condition);
+ }
+
+ // Moves zero to a register rd, if the condition rs2 is nonzero, otherwise moves rs1 to rd.
+ void czero_nez(Register rd, Register rs1_value, Register rs2_condition) {
+ czero(rd, rs1_value, rs2_condition);
+ }
+
// -------------- ZCB Instruction Definitions --------------
// Zcb additional C instructions
private:
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
index eb715227f7e..b7edd3d231f 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
@@ -870,6 +870,7 @@ void LIR_Assembler::emit_op3(LIR_Op3* op) {
}
}
+// Consider using cmov (Zicond)
void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
Label label;
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
index 49efb619093..122ea7352fe 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
@@ -116,10 +116,10 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg,
// Handle existing monitor.
bind(object_has_monitor);
- // Try to CAS owner (no owner => current thread's _lock_id).
+ // Try to CAS owner (no owner => current thread's _monitor_owner_id).
add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value));
Register tid = tmp4Reg;
- ld(tid, Address(xthread, JavaThread::lock_id_offset()));
+ ld(tid, Address(xthread, JavaThread::monitor_owner_id_offset()));
cmpxchg(/*memory address*/tmp, /*expected value*/zr, /*new value*/tid, Assembler::int64,
Assembler::aq, Assembler::rl, /*result*/tmp3Reg); // cas succeeds if tmp3Reg == zr(expected)
@@ -400,9 +400,9 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box,
// Compute owner address.
la(tmp2_owner_addr, owner_address);
- // Try to CAS owner (no owner => current thread's _lock_id).
+ // Try to CAS owner (no owner => current thread's _monitor_owner_id).
Register tid = tmp4;
- ld(tid, Address(xthread, JavaThread::lock_id_offset()));
+ ld(tid, Address(xthread, JavaThread::monitor_owner_id_offset()));
cmpxchg(/*addr*/ tmp2_owner_addr, /*expected*/ zr, /*new*/ tid, Assembler::int64,
/*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_owner);
beqz(tmp3_owner, monitor_locked);
@@ -2003,10 +2003,48 @@ void C2_MacroAssembler::enc_cmpEqNe_imm0_branch(int cmpFlag, Register op1, Label
}
void C2_MacroAssembler::enc_cmove(int cmpFlag, Register op1, Register op2, Register dst, Register src) {
- Label L;
- cmp_branch(cmpFlag ^ (1 << neg_cond_bits), op1, op2, L);
- mv(dst, src);
- bind(L);
+ bool is_unsigned = (cmpFlag & unsigned_branch_mask) == unsigned_branch_mask;
+ int op_select = cmpFlag & (~unsigned_branch_mask);
+
+ switch (op_select) {
+ case BoolTest::eq:
+ cmov_eq(op1, op2, dst, src);
+ break;
+ case BoolTest::ne:
+ cmov_ne(op1, op2, dst, src);
+ break;
+ case BoolTest::le:
+ if (is_unsigned) {
+ cmov_leu(op1, op2, dst, src);
+ } else {
+ cmov_le(op1, op2, dst, src);
+ }
+ break;
+ case BoolTest::ge:
+ if (is_unsigned) {
+ cmov_geu(op1, op2, dst, src);
+ } else {
+ cmov_ge(op1, op2, dst, src);
+ }
+ break;
+ case BoolTest::lt:
+ if (is_unsigned) {
+ cmov_ltu(op1, op2, dst, src);
+ } else {
+ cmov_lt(op1, op2, dst, src);
+ }
+ break;
+ case BoolTest::gt:
+ if (is_unsigned) {
+ cmov_gtu(op1, op2, dst, src);
+ } else {
+ cmov_gt(op1, op2, dst, src);
+ }
+ break;
+ default:
+ assert(false, "unsupported compare condition");
+ ShouldNotReachHere();
+ }
}
// Set dst to NaN if any NaN input.
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
index 2d14f98780d..a8eb0df419c 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
@@ -98,7 +98,6 @@
// refer to conditional_branches and float_conditional_branches
static const int bool_test_bits = 3;
- static const int neg_cond_bits = 2;
static const int unsigned_branch_mask = 1 << bool_test_bits;
static const int double_branch_mask = 1 << bool_test_bits;
diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp b/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp
index f503cb762e7..d15b3aa31f9 100644
--- a/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp
@@ -78,9 +78,14 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LI
LIR_Opr result = gen->new_register(T_INT);
__ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), tmp1, tmp2, result));
+
+ if (ShenandoahCardBarrier) {
+ post_barrier(access, access.resolved_addr(), new_value.result());
+ }
return result;
}
}
+
return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
}
@@ -105,6 +110,9 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt
pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
result /* pre_val */);
}
+ if (ShenandoahCardBarrier) {
+ post_barrier(access, access.resolved_addr(), result);
+ }
}
return result;
diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp
index cc73d14a756..257d445f011 100644
--- a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp
@@ -32,6 +32,7 @@
#include "gc/shenandoah/shenandoahRuntime.hpp"
#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/mode/shenandoahMode.hpp"
#include "interpreter/interpreter.hpp"
#include "interpreter/interp_masm.hpp"
#include "runtime/javaThread.hpp"
@@ -81,6 +82,13 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
}
}
+void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+ Register start, Register count, Register tmp, RegSet saved_regs) {
+ if (ShenandoahCardBarrier && is_oop) {
+ gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp, saved_regs);
+ }
+}
+
void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
Register obj,
Register pre_val,
@@ -382,6 +390,27 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm,
}
}
+void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
+ assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");
+
+ __ srli(obj, obj, CardTable::card_shift());
+
+ assert(CardTable::dirty_card_val() == 0, "must be");
+
+ __ load_byte_map_base(t1);
+ __ add(t1, obj, t1);
+
+ if (UseCondCardMark) {
+ Label L_already_dirty;
+ __ lbu(t0, Address(t1));
+ __ beqz(t0, L_already_dirty);
+ __ sb(zr, Address(t1));
+ __ bind(L_already_dirty);
+ } else {
+ __ sb(zr, Address(t1));
+ }
+}
+
void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
bool on_oop = is_reference_type(type);
@@ -407,16 +436,12 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet
val != noreg /* tosca_live */,
false /* expand_call */);
- if (val == noreg) {
- BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg);
- } else {
- // Barrier needs uncompressed oop for region cross check.
- Register new_val = val;
- if (UseCompressedOops) {
- new_val = t1;
- __ mv(new_val, val);
- }
- BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
+ BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
+
+ bool in_heap = (decorators & IN_HEAP) != 0;
+ bool needs_post_barrier = (val != noreg) && in_heap && ShenandoahCardBarrier;
+ if (needs_post_barrier) {
+ store_check(masm, tmp3);
}
}
@@ -524,6 +549,37 @@ void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
__ bind(done);
}
+void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+ Register start, Register count, Register tmp, RegSet saved_regs) {
+ assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");
+
+ Label L_loop, L_done;
+ const Register end = count;
+
+ // Zero count? Nothing to do.
+ __ beqz(count, L_done);
+
+ // end = start + count << LogBytesPerHeapOop
+ // last element address to make inclusive
+ __ shadd(end, count, start, tmp, LogBytesPerHeapOop);
+ __ sub(end, end, BytesPerHeapOop);
+ __ srli(start, start, CardTable::card_shift());
+ __ srli(end, end, CardTable::card_shift());
+
+ // number of bytes to copy
+ __ sub(count, end, start);
+
+ __ load_byte_map_base(tmp);
+ __ add(start, start, tmp);
+
+ __ bind(L_loop);
+ __ add(tmp, start, count);
+ __ sb(zr, Address(tmp));
+ __ sub(count, count, 1);
+ __ bgez(count, L_loop);
+ __ bind(L_done);
+}
+
#undef __
#ifdef COMPILER1
diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp
index bfdea7f607e..7d12cc8cbb6 100644
--- a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp
@@ -57,10 +57,16 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
bool tosca_live,
bool expand_call);
+ void store_check(MacroAssembler* masm, Register obj);
+
void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg);
void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg);
void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators);
+ void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+ Register start, Register count,
+ Register tmp, RegSet saved_regs);
+
public:
virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_data_patch; }
@@ -75,6 +81,9 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
Register src, Register dst, Register count, RegSet saved_regs);
+ virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+ Register start, Register count, Register tmp, RegSet saved_regs);
+
virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp2);
virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
diff --git a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp
index cd83eafcaeb..dde2f1f131f 100644
--- a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp
@@ -286,7 +286,7 @@ void ZBarrierSetAssembler::store_barrier_medium(MacroAssembler* masm,
__ relocate(barrier_Relocation::spec(), [&] {
__ li16u(rtmp1, barrier_Relocation::unpatched);
}, ZBarrierRelocationFormatStoreGoodBits);
- __ cmpxchg_weak(rtmp2, zr, rtmp1,
+ __ weak_cmpxchg(rtmp2, zr, rtmp1,
Assembler::int64,
Assembler::relaxed /* acquire */, Assembler::relaxed /* release */,
rtmp3);
diff --git a/src/hotspot/cpu/riscv/globals_riscv.hpp b/src/hotspot/cpu/riscv/globals_riscv.hpp
index ffbb7c58911..6772fae50ca 100644
--- a/src/hotspot/cpu/riscv/globals_riscv.hpp
+++ b/src/hotspot/cpu/riscv/globals_riscv.hpp
@@ -110,6 +110,7 @@ define_pd_global(intx, InlineSmallCode, 1000);
product(bool, UseZicbom, false, EXPERIMENTAL, "Use Zicbom instructions") \
product(bool, UseZicbop, false, EXPERIMENTAL, "Use Zicbop instructions") \
product(bool, UseZicboz, false, EXPERIMENTAL, "Use Zicboz instructions") \
+ product(bool, UseZicond, false, DIAGNOSTIC, "Use Zicond instructions") \
product(bool, UseZihintpause, false, EXPERIMENTAL, \
"Use Zihintpause instructions") \
product(bool, UseZtso, false, EXPERIMENTAL, "Assume Ztso memory model") \
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
index cf3c851a7ec..44b806834f9 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
@@ -1128,6 +1128,147 @@ void MacroAssembler::wrap_label(Register r1, Register r2, Label &L,
#undef INSN
+// cmov
+void MacroAssembler::cmov_eq(Register cmp1, Register cmp2, Register dst, Register src) {
+ if (UseZicond) {
+ xorr(t0, cmp1, cmp2);
+ czero_eqz(dst, dst, t0);
+ czero_nez(t0 , src, t0);
+ orr(dst, dst, t0);
+ return;
+ }
+ Label no_set;
+ bne(cmp1, cmp2, no_set);
+ mv(dst, src);
+ bind(no_set);
+}
+
+void MacroAssembler::cmov_ne(Register cmp1, Register cmp2, Register dst, Register src) {
+ if (UseZicond) {
+ xorr(t0, cmp1, cmp2);
+ czero_nez(dst, dst, t0);
+ czero_eqz(t0 , src, t0);
+ orr(dst, dst, t0);
+ return;
+ }
+ Label no_set;
+ beq(cmp1, cmp2, no_set);
+ mv(dst, src);
+ bind(no_set);
+}
+
+void MacroAssembler::cmov_le(Register cmp1, Register cmp2, Register dst, Register src) {
+ if (UseZicond) {
+ slt(t0, cmp2, cmp1);
+ czero_eqz(dst, dst, t0);
+ czero_nez(t0, src, t0);
+ orr(dst, dst, t0);
+ return;
+ }
+ Label no_set;
+ bgt(cmp1, cmp2, no_set);
+ mv(dst, src);
+ bind(no_set);
+}
+
+void MacroAssembler::cmov_leu(Register cmp1, Register cmp2, Register dst, Register src) {
+ if (UseZicond) {
+ sltu(t0, cmp2, cmp1);
+ czero_eqz(dst, dst, t0);
+ czero_nez(t0, src, t0);
+ orr(dst, dst, t0);
+ return;
+ }
+ Label no_set;
+ bgtu(cmp1, cmp2, no_set);
+ mv(dst, src);
+ bind(no_set);
+}
+
+void MacroAssembler::cmov_ge(Register cmp1, Register cmp2, Register dst, Register src) {
+ if (UseZicond) {
+ slt(t0, cmp1, cmp2);
+ czero_eqz(dst, dst, t0);
+ czero_nez(t0, src, t0);
+ orr(dst, dst, t0);
+ return;
+ }
+ Label no_set;
+ blt(cmp1, cmp2, no_set);
+ mv(dst, src);
+ bind(no_set);
+}
+
+void MacroAssembler::cmov_geu(Register cmp1, Register cmp2, Register dst, Register src) {
+ if (UseZicond) {
+ sltu(t0, cmp1, cmp2);
+ czero_eqz(dst, dst, t0);
+ czero_nez(t0, src, t0);
+ orr(dst, dst, t0);
+ return;
+ }
+ Label no_set;
+ bltu(cmp1, cmp2, no_set);
+ mv(dst, src);
+ bind(no_set);
+}
+
+void MacroAssembler::cmov_lt(Register cmp1, Register cmp2, Register dst, Register src) {
+ if (UseZicond) {
+ slt(t0, cmp1, cmp2);
+ czero_nez(dst, dst, t0);
+ czero_eqz(t0, src, t0);
+ orr(dst, dst, t0);
+ return;
+ }
+ Label no_set;
+ bge(cmp1, cmp2, no_set);
+ mv(dst, src);
+ bind(no_set);
+}
+
+void MacroAssembler::cmov_ltu(Register cmp1, Register cmp2, Register dst, Register src) {
+ if (UseZicond) {
+ sltu(t0, cmp1, cmp2);
+ czero_nez(dst, dst, t0);
+ czero_eqz(t0, src, t0);
+ orr(dst, dst, t0);
+ return;
+ }
+ Label no_set;
+ bgeu(cmp1, cmp2, no_set);
+ mv(dst, src);
+ bind(no_set);
+}
+
+void MacroAssembler::cmov_gt(Register cmp1, Register cmp2, Register dst, Register src) {
+ if (UseZicond) {
+ slt(t0, cmp2, cmp1);
+ czero_nez(dst, dst, t0);
+ czero_eqz(t0, src, t0);
+ orr(dst, dst, t0);
+ return;
+ }
+ Label no_set;
+ ble(cmp1, cmp2, no_set);
+ mv(dst, src);
+ bind(no_set);
+}
+
+void MacroAssembler::cmov_gtu(Register cmp1, Register cmp2, Register dst, Register src) {
+ if (UseZicond) {
+ sltu(t0, cmp2, cmp1);
+ czero_nez(dst, dst, t0);
+ czero_eqz(t0, src, t0);
+ orr(dst, dst, t0);
+ return;
+ }
+ Label no_set;
+ bleu(cmp1, cmp2, no_set);
+ mv(dst, src);
+ bind(no_set);
+}
+
// Float compare branch instructions
#define INSN(NAME, FLOATCMP, BRANCH) \
@@ -2320,27 +2461,9 @@ void MacroAssembler::load_long_misaligned(Register dst, Address src, Register tm
}
}
-
-// reverse bytes in halfword in lower 16 bits and sign-extend
-// Rd[15:0] = Rs[7:0] Rs[15:8] (sign-extend to 64 bits)
-void MacroAssembler::revb_h_h(Register Rd, Register Rs, Register tmp) {
- if (UseZbb) {
- rev8(Rd, Rs);
- srai(Rd, Rd, 48);
- return;
- }
- assert_different_registers(Rs, tmp);
- assert_different_registers(Rd, tmp);
- srli(tmp, Rs, 8);
- andi(tmp, tmp, 0xFF);
- slli(Rd, Rs, 56);
- srai(Rd, Rd, 48); // sign-extend
- orr(Rd, Rd, tmp);
-}
-
-// reverse bytes in lower word and sign-extend
-// Rd[31:0] = Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24] (sign-extend to 64 bits)
-void MacroAssembler::revb_w_w(Register Rd, Register Rs, Register tmp1, Register tmp2) {
+// reverse bytes in lower word, sign-extend
+// Rd[32:0] = Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24]
+void MacroAssembler::revbw(Register Rd, Register Rs, Register tmp1, Register tmp2) {
if (UseZbb) {
rev8(Rd, Rs);
srai(Rd, Rd, 32);
@@ -2348,99 +2471,18 @@ void MacroAssembler::revb_w_w(Register Rd, Register Rs, Register tmp1, Register
}
assert_different_registers(Rs, tmp1, tmp2);
assert_different_registers(Rd, tmp1, tmp2);
- revb_h_w_u(Rd, Rs, tmp1, tmp2);
- slli(tmp2, Rd, 48);
- srai(tmp2, tmp2, 32); // sign-extend
- srli(Rd, Rd, 16);
- orr(Rd, Rd, tmp2);
-}
-
-// reverse bytes in halfword in lower 16 bits and zero-extend
-// Rd[15:0] = Rs[7:0] Rs[15:8] (zero-extend to 64 bits)
-void MacroAssembler::revb_h_h_u(Register Rd, Register Rs, Register tmp) {
- if (UseZbb) {
- rev8(Rd, Rs);
- srli(Rd, Rd, 48);
- return;
- }
- assert_different_registers(Rs, tmp);
- assert_different_registers(Rd, tmp);
- srli(tmp, Rs, 8);
- andi(tmp, tmp, 0xFF);
- andi(Rd, Rs, 0xFF);
- slli(Rd, Rd, 8);
- orr(Rd, Rd, tmp);
-}
-
-// reverse bytes in halfwords in lower 32 bits and zero-extend
-// Rd[31:0] = Rs[23:16] Rs[31:24] Rs[7:0] Rs[15:8] (zero-extend to 64 bits)
-void MacroAssembler::revb_h_w_u(Register Rd, Register Rs, Register tmp1, Register tmp2) {
- if (UseZbb) {
- rev8(Rd, Rs);
- rori(Rd, Rd, 32);
- roriw(Rd, Rd, 16);
- zero_extend(Rd, Rd, 32);
- return;
- }
- assert_different_registers(Rs, tmp1, tmp2);
- assert_different_registers(Rd, tmp1, tmp2);
- srli(tmp2, Rs, 16);
- revb_h_h_u(tmp2, tmp2, tmp1);
- revb_h_h_u(Rd, Rs, tmp1);
- slli(tmp2, tmp2, 16);
- orr(Rd, Rd, tmp2);
-}
-
-// This method is only used for revb_h
-// Rd = Rs[47:0] Rs[55:48] Rs[63:56]
-void MacroAssembler::revb_h_helper(Register Rd, Register Rs, Register tmp1, Register tmp2) {
- assert_different_registers(Rs, tmp1, tmp2);
- assert_different_registers(Rd, tmp1);
- srli(tmp1, Rs, 48);
- andi(tmp2, tmp1, 0xFF);
- slli(tmp2, tmp2, 8);
- srli(tmp1, tmp1, 8);
- orr(tmp1, tmp1, tmp2);
- slli(Rd, Rs, 16);
- orr(Rd, Rd, tmp1);
-}
-
-// reverse bytes in each halfword
-// Rd[63:0] = Rs[55:48] Rs[63:56] Rs[39:32] Rs[47:40] Rs[23:16] Rs[31:24] Rs[7:0] Rs[15:8]
-void MacroAssembler::revb_h(Register Rd, Register Rs, Register tmp1, Register tmp2) {
- if (UseZbb) {
- assert_different_registers(Rs, tmp1);
- assert_different_registers(Rd, tmp1);
- rev8(Rd, Rs);
- zero_extend(tmp1, Rd, 32);
- roriw(tmp1, tmp1, 16);
- slli(tmp1, tmp1, 32);
- srli(Rd, Rd, 32);
- roriw(Rd, Rd, 16);
- zero_extend(Rd, Rd, 32);
- orr(Rd, Rd, tmp1);
- return;
- }
- assert_different_registers(Rs, tmp1, tmp2);
- assert_different_registers(Rd, tmp1, tmp2);
- revb_h_helper(Rd, Rs, tmp1, tmp2);
- for (int i = 0; i < 3; ++i) {
- revb_h_helper(Rd, Rd, tmp1, tmp2);
- }
-}
-
-// reverse bytes in each word
-// Rd[63:0] = Rs[39:32] Rs[47:40] Rs[55:48] Rs[63:56] Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24]
-void MacroAssembler::revb_w(Register Rd, Register Rs, Register tmp1, Register tmp2) {
- if (UseZbb) {
- rev8(Rd, Rs);
- rori(Rd, Rd, 32);
- return;
+ andi(tmp1, Rs, 0xFF);
+ slli(tmp1, tmp1, 8);
+ for (int step = 8; step < 24; step += 8) {
+ srli(tmp2, Rs, step);
+ andi(tmp2, tmp2, 0xFF);
+ orr(tmp1, tmp1, tmp2);
+ slli(tmp1, tmp1, 8);
}
- assert_different_registers(Rs, tmp1, tmp2);
- assert_different_registers(Rd, tmp1, tmp2);
- revb(Rd, Rs, tmp1, tmp2);
- ror_imm(Rd, Rd, 32);
+ srli(Rd, Rs, 24);
+ andi(Rd, Rd, 0xFF);
+ orr(Rd, tmp1, Rd);
+ sign_extend(Rd, Rd, 32);
}
// reverse bytes in doubleword
@@ -3534,7 +3576,7 @@ void MacroAssembler::cmpxchg(Register addr, Register expected,
bind(done);
}
-void MacroAssembler::cmpxchg_weak(Register addr, Register expected,
+void MacroAssembler::weak_cmpxchg(Register addr, Register expected,
Register new_val,
enum operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
index 54f7127106b..0d28eaaf1f0 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
@@ -626,6 +626,17 @@ class MacroAssembler: public Assembler {
void bltz(Register Rs, const address dest);
void bgtz(Register Rs, const address dest);
+ void cmov_eq(Register cmp1, Register cmp2, Register dst, Register src);
+ void cmov_ne(Register cmp1, Register cmp2, Register dst, Register src);
+ void cmov_le(Register cmp1, Register cmp2, Register dst, Register src);
+ void cmov_leu(Register cmp1, Register cmp2, Register dst, Register src);
+ void cmov_ge(Register cmp1, Register cmp2, Register dst, Register src);
+ void cmov_geu(Register cmp1, Register cmp2, Register dst, Register src);
+ void cmov_lt(Register cmp1, Register cmp2, Register dst, Register src);
+ void cmov_ltu(Register cmp1, Register cmp2, Register dst, Register src);
+ void cmov_gt(Register cmp1, Register cmp2, Register dst, Register src);
+ void cmov_gtu(Register cmp1, Register cmp2, Register dst, Register src);
+
public:
// We try to follow risc-v asm menomics.
// But as we don't layout a reachable GOT,
@@ -901,15 +912,9 @@ class MacroAssembler: public Assembler {
void andn(Register Rd, Register Rs1, Register Rs2);
void orn(Register Rd, Register Rs1, Register Rs2);
- // revb
- void revb_h_h(Register Rd, Register Rs, Register tmp = t0); // reverse bytes in halfword in lower 16 bits, sign-extend
- void revb_w_w(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2 = t1); // reverse bytes in lower word, sign-extend
- void revb_h_h_u(Register Rd, Register Rs, Register tmp = t0); // reverse bytes in halfword in lower 16 bits, zero-extend
- void revb_h_w_u(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2 = t1); // reverse bytes in halfwords in lower 32 bits, zero-extend
- void revb_h_helper(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2= t1); // reverse bytes in upper 16 bits (48:63) and move to lower
- void revb_h(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2= t1); // reverse bytes in each halfword
- void revb_w(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2= t1); // reverse bytes in each word
- void revb(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2 = t1); // reverse bytes in doubleword
+ // reverse bytes
+ void revbw(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2= t1); // reverse bytes in lower word, sign-extend
+ void revb(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2 = t1); // reverse bytes in doubleword
void ror_imm(Register dst, Register src, uint32_t shift, Register tmp = t0);
void rolw_imm(Register dst, Register src, uint32_t, Register tmp = t0);
@@ -1141,7 +1146,7 @@ class MacroAssembler: public Assembler {
enum operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result, bool result_as_bool = false);
- void cmpxchg_weak(Register addr, Register expected,
+ void weak_cmpxchg(Register addr, Register expected,
Register new_val,
enum operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad
index d0085125c76..32019251bbe 100644
--- a/src/hotspot/cpu/riscv/riscv.ad
+++ b/src/hotspot/cpu/riscv/riscv.ad
@@ -5744,7 +5744,7 @@ instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iReg
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
- "cmpxchg_weak $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapB"
%}
@@ -5767,7 +5767,7 @@ instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iReg
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
- "cmpxchg_weak $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapS"
%}
@@ -5787,12 +5787,12 @@ instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI ne
ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
format %{
- "cmpxchg_weak $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapI"
%}
ins_encode %{
- __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32,
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32,
/*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
%}
@@ -5806,12 +5806,12 @@ instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL ne
ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
format %{
- "cmpxchg_weak $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapL"
%}
ins_encode %{
- __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
/*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
%}
@@ -5826,12 +5826,12 @@ instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN ne
ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 4);
format %{
- "cmpxchg_weak $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapN"
%}
ins_encode %{
- __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32,
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32,
/*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
%}
@@ -5846,12 +5846,12 @@ instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
format %{
- "cmpxchg_weak $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapP"
%}
ins_encode %{
- __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
/*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
%}
@@ -5870,7 +5870,7 @@ instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, i
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
- "cmpxchg_weak_acq $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapBAcq"
%}
@@ -5895,7 +5895,7 @@ instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, i
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
- "cmpxchg_weak_acq $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapSAcq"
%}
@@ -5917,12 +5917,12 @@ instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI
ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
format %{
- "cmpxchg_weak_acq $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapIAcq"
%}
ins_encode %{
- __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32,
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32,
/*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
%}
@@ -5938,12 +5938,12 @@ instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL
ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
format %{
- "cmpxchg_weak_acq $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapLAcq"
%}
ins_encode %{
- __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
/*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
%}
@@ -5959,12 +5959,12 @@ instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN
ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 4);
format %{
- "cmpxchg_weak_acq $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapNAcq"
%}
ins_encode %{
- __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32,
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32,
/*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
%}
@@ -5980,12 +5980,12 @@ instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP
ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
format %{
- "cmpxchg_weak_acq $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"\t# $res == 1 when success, #@weakCompareAndSwapPAcq"
%}
ins_encode %{
- __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
/*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
%}
diff --git a/src/hotspot/cpu/riscv/riscv_b.ad b/src/hotspot/cpu/riscv/riscv_b.ad
index 92e616a3063..6b7645858d8 100644
--- a/src/hotspot/cpu/riscv/riscv_b.ad
+++ b/src/hotspot/cpu/riscv/riscv_b.ad
@@ -181,11 +181,15 @@ instruct bytes_reverse_int_b(iRegINoSp dst, iRegIorL2I src) %{
match(Set dst (ReverseBytesI src));
ins_cost(ALU_COST * 2);
- format %{ "revb_w_w $dst, $src\t#@bytes_reverse_int_b" %}
+ format %{
+ "rev8 $dst, $src\t#@bytes_reverse_int_b\t\n"
+ "srai $dst, $dst, 32\t\n"
+ %}
ins_encode %{
assert(UseZbb, "must be");
- __ revb_w_w(as_Register($dst$$reg), as_Register($src$$reg));
+ __ rev8(as_Register($dst$$reg), as_Register($src$$reg));
+ __ srai(as_Register($dst$$reg), as_Register($dst$$reg), 32);
%}
ins_pipe(ialu_reg);
@@ -209,11 +213,15 @@ instruct bytes_reverse_unsigned_short_b(iRegINoSp dst, iRegIorL2I src) %{
match(Set dst (ReverseBytesUS src));
ins_cost(ALU_COST * 2);
- format %{ "revb_h_h_u $dst, $src\t#@bytes_reverse_unsigned_short_b" %}
+ format %{
+ "rev8 $dst, $src\t#@bytes_reverse_unsigned_short_b\t\n"
+ "srli $dst, $dst, 48\t\n"
+ %}
ins_encode %{
assert(UseZbb, "must be");
- __ revb_h_h_u(as_Register($dst$$reg), as_Register($src$$reg));
+ __ rev8(as_Register($dst$$reg), as_Register($src$$reg));
+ __ srli(as_Register($dst$$reg), as_Register($dst$$reg), 48);
%}
ins_pipe(ialu_reg);
@@ -223,11 +231,15 @@ instruct bytes_reverse_short_b(iRegINoSp dst, iRegIorL2I src) %{
match(Set dst (ReverseBytesS src));
ins_cost(ALU_COST * 2);
- format %{ "revb_h_h $dst, $src\t#@bytes_reverse_short_b" %}
+ format %{
+ "rev8 $dst, $src\t#@bytes_reverse_short_b\t\n"
+ "srai $dst, $dst, 48\t\n"
+ %}
ins_encode %{
assert(UseZbb, "must be");
- __ revb_h_h(as_Register($dst$$reg), as_Register($src$$reg));
+ __ rev8(as_Register($dst$$reg), as_Register($src$$reg));
+ __ srai(as_Register($dst$$reg), as_Register($dst$$reg), 48);
%}
ins_pipe(ialu_reg);
diff --git a/src/hotspot/cpu/riscv/templateTable_riscv.cpp b/src/hotspot/cpu/riscv/templateTable_riscv.cpp
index 62dc952bde0..e583579d618 100644
--- a/src/hotspot/cpu/riscv/templateTable_riscv.cpp
+++ b/src/hotspot/cpu/riscv/templateTable_riscv.cpp
@@ -658,8 +658,12 @@ void TemplateTable::aload() {
}
void TemplateTable::locals_index_wide(Register reg) {
- __ lhu(reg, at_bcp(2));
- __ revb_h_h_u(reg, reg); // reverse bytes in half-word and zero-extend
+ assert_different_registers(reg, t1);
+ // Convert the 16-bit value into native byte-ordering and zero-extend
+ __ lbu(reg, at_bcp(2));
+ __ lbu(t1, at_bcp(3));
+ __ slli(reg, reg, 8);
+ __ orr(reg, reg, t1);
__ neg(reg, reg);
}
@@ -671,8 +675,12 @@ void TemplateTable::wide_iload() {
void TemplateTable::wide_lload() {
transition(vtos, ltos);
- __ lhu(x11, at_bcp(2));
- __ revb_h_h_u(x11, x11); // reverse bytes in half-word and zero-extend
+ // Convert the 16-bit value into native byte-ordering and zero-extend
+ __ lbu(x11, at_bcp(2));
+ __ lbu(t1, at_bcp(3));
+ __ slli(x11, x11, 8);
+ __ orr(x11, x11, t1);
+
__ slli(x11, x11, LogBytesPerWord);
__ sub(x11, xlocals, x11);
__ ld(x10, Address(x11, Interpreter::local_offset_in_bytes(1)));
@@ -686,8 +694,12 @@ void TemplateTable::wide_fload() {
void TemplateTable::wide_dload() {
transition(vtos, dtos);
- __ lhu(x11, at_bcp(2));
- __ revb_h_h_u(x11, x11); // reverse bytes in half-word and zero-extend
+ // Convert the 16-bit value into native byte-ordering and zero-extend
+ __ lbu(x11, at_bcp(2));
+ __ lbu(t1, at_bcp(3));
+ __ slli(x11, x11, 8);
+ __ orr(x11, x11, t1);
+
__ slli(x11, x11, LogBytesPerWord);
__ sub(x11, xlocals, x11);
__ fld(f10, Address(x11, Interpreter::local_offset_in_bytes(1)));
@@ -1471,12 +1483,14 @@ void TemplateTable::iinc() {
void TemplateTable::wide_iinc() {
transition(vtos, vtos);
- __ lwu(x11, at_bcp(2)); // get constant and index
- __ revb_h_w_u(x11, x11); // reverse bytes in half-word (32bit) and zero-extend
- __ zero_extend(x12, x11, 16);
- __ neg(x12, x12);
- __ slli(x11, x11, 32);
- __ srai(x11, x11, 48);
+ // get constant
+ // Convert the 16-bit value into native byte-ordering and sign-extend
+ __ lb(x11, at_bcp(4));
+ __ lbu(t1, at_bcp(5));
+ __ slli(x11, x11, 8);
+ __ orr(x11, x11, t1);
+
+ locals_index_wide(x12);
__ ld(x10, iaddress(x12, t0, _masm));
__ addw(x10, x10, x11);
__ sd(x10, iaddress(x12, t0, _masm));
@@ -1621,13 +1635,14 @@ void TemplateTable::branch(bool is_jsr, bool is_wide) {
// load branch displacement
if (!is_wide) {
+ // Convert the 16-bit value into native byte-ordering and sign-extend
__ lb(x12, at_bcp(1));
__ lbu(t1, at_bcp(2));
__ slli(x12, x12, 8);
- __ add(x12, x12, t1);
+ __ orr(x12, x12, t1);
} else {
__ lwu(x12, at_bcp(1));
- __ revb_w_w(x12, x12); // reverse bytes in word and sign-extend
+ __ revbw(x12, x12);
}
// Handle all the JSR stuff here, then exit.
@@ -1892,8 +1907,8 @@ void TemplateTable::tableswitch() {
// load lo & hi
__ lwu(x12, Address(x11, BytesPerInt));
__ lwu(x13, Address(x11, 2 * BytesPerInt));
- __ revb_w_w(x12, x12); // reverse bytes in word (32bit) and sign-extend
- __ revb_w_w(x13, x13); // reverse bytes in word (32bit) and sign-extend
+ __ revbw(x12, x12);
+ __ revbw(x13, x13);
// check against lo & hi
__ blt(x10, x12, default_case);
__ bgt(x10, x13, default_case);
@@ -1904,7 +1919,7 @@ void TemplateTable::tableswitch() {
__ profile_switch_case(x10, x11, x12);
// continue execution
__ bind(continue_execution);
- __ revb_w_w(x13, x13); // reverse bytes in word (32bit) and sign-extend
+ __ revbw(x13, x13);
__ add(xbcp, xbcp, x13);
__ load_unsigned_byte(t0, Address(xbcp));
__ dispatch_only(vtos, /*generate_poll*/true);
@@ -1924,7 +1939,7 @@ void TemplateTable::fast_linearswitch() {
transition(itos, vtos);
Label loop_entry, loop, found, continue_execution;
// bswap x10 so we can avoid bswapping the table entries
- __ revb_w_w(x10, x10); // reverse bytes in word (32bit) and sign-extend
+ __ revbw(x10, x10);
// align xbcp
__ la(x9, at_bcp(BytesPerInt)); // btw: should be able to get rid of
// this instruction (change offsets
@@ -1932,7 +1947,10 @@ void TemplateTable::fast_linearswitch() {
__ andi(x9, x9, -BytesPerInt);
// set counter
__ lwu(x11, Address(x9, BytesPerInt));
- __ revb_w(x11, x11);
+ // Convert the 32-bit npairs (number of pairs) into native byte-ordering
+ // We can use sign-extension here because npairs must be greater than or
+ // equal to 0 per JVM spec on 'lookupswitch' bytecode.
+ __ revbw(x11, x11);
__ j(loop_entry);
// table search
__ bind(loop);
@@ -1953,7 +1971,7 @@ void TemplateTable::fast_linearswitch() {
__ profile_switch_case(x11, x10, x9);
// continue execution
__ bind(continue_execution);
- __ revb_w_w(x13, x13); // reverse bytes in word (32bit) and sign-extend
+ __ revbw(x13, x13);
__ add(xbcp, xbcp, x13);
__ lbu(t0, Address(xbcp, 0));
__ dispatch_only(vtos, /*generate_poll*/true);
@@ -2005,8 +2023,10 @@ void TemplateTable::fast_binaryswitch() {
__ mv(i, zr); // i = 0
__ lwu(j, Address(array, -BytesPerInt)); // j = length(array)
- // Convert j into native byteordering
- __ revb_w(j, j);
+ // Convert the 32-bit npairs (number of pairs) into native byte-ordering
+ // We can use sign-extension here because npairs must be greater than or
+ // equal to 0 per JVM spec on 'lookupswitch' bytecode.
+ __ revbw(j, j);
// And start
Label entry;
@@ -2024,7 +2044,7 @@ void TemplateTable::fast_binaryswitch() {
// Convert array[h].match to native byte-ordering before compare
__ shadd(temp, h, array, temp, 3);
__ lwu(temp, Address(temp, 0));
- __ revb_w_w(temp, temp); // reverse bytes in word (32bit) and sign-extend
+ __ revbw(temp, temp);
Label L_done, L_greater;
__ bge(key, temp, L_greater);
@@ -2047,14 +2067,14 @@ void TemplateTable::fast_binaryswitch() {
// Convert array[i].match to native byte-ordering before compare
__ shadd(temp, i, array, temp, 3);
__ lwu(temp, Address(temp, 0));
- __ revb_w_w(temp, temp); // reverse bytes in word (32bit) and sign-extend
+ __ revbw(temp, temp);
__ bne(key, temp, default_case);
// entry found -> j = offset
__ shadd(temp, i, array, temp, 3);
__ lwu(j, Address(temp, BytesPerInt));
__ profile_switch_case(i, key, array);
- __ revb_w_w(j, j); // reverse bytes in word (32bit) and sign-extend
+ __ revbw(j, j);
__ add(temp, xbcp, j);
__ load_unsigned_byte(t0, Address(temp, 0));
@@ -2067,7 +2087,7 @@ void TemplateTable::fast_binaryswitch() {
__ bind(default_case);
__ profile_switch_default(i);
__ lwu(j, Address(array, -2 * BytesPerInt));
- __ revb_w_w(j, j); // reverse bytes in word (32bit) and sign-extend
+ __ revbw(j, j);
__ add(temp, xbcp, j);
__ load_unsigned_byte(t0, Address(temp, 0));
diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.hpp b/src/hotspot/cpu/riscv/vm_version_riscv.hpp
index 8fdde0094f4..e08838c3a6f 100644
--- a/src/hotspot/cpu/riscv/vm_version_riscv.hpp
+++ b/src/hotspot/cpu/riscv/vm_version_riscv.hpp
@@ -116,6 +116,8 @@ class VM_Version : public Abstract_VM_Version {
//
// Zfh Half-Precision Floating-Point instructions
//
+ // Zicond Conditional operations
+ //
// Zicsr Control and Status Register (CSR) Instructions
// Zifencei Instruction-Fetch Fence
// Zic64b Cache blocks must be 64 bytes in size, naturally aligned in the address space.
@@ -164,6 +166,7 @@ class VM_Version : public Abstract_VM_Version {
decl(ext_Zvbb , "Zvbb" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZvbb)) \
decl(ext_Zvfh , "Zvfh" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZvfh)) \
decl(ext_Zvkn , "Zvkn" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZvkn)) \
+ decl(ext_Zicond , "Zicond" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicond)) \
decl(mvendorid , "VendorId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
decl(marchid , "ArchId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
decl(mimpid , "ImpId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
@@ -223,6 +226,7 @@ class VM_Version : public Abstract_VM_Version {
RV_ENABLE_EXTENSION(UseZicbom) \
RV_ENABLE_EXTENSION(UseZicbop) \
RV_ENABLE_EXTENSION(UseZicboz) \
+ RV_ENABLE_EXTENSION(UseZicond) \
RV_ENABLE_EXTENSION(UseZihintpause) \
static void useRVA23U64Profile();
diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
index 213aa5efe1e..bb0494dc478 100644
--- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
@@ -1532,8 +1532,12 @@ void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr
// cpu register - constant
jint c = right->as_constant_ptr()->as_jint();
switch (code) {
- case lir_add: __ z_agfi(lreg, c); break;
- case lir_sub: __ z_agfi(lreg, -c); break; // note: -min_jint == min_jint
+ case lir_add:
+ __ add2reg_32(lreg, c);
+ break;
+ case lir_sub:
+ __ add2reg_32(lreg, java_negate(c));
+ break;
case lir_mul: __ z_msfi(lreg, c); break;
default: ShouldNotReachHere();
}
@@ -2539,13 +2543,11 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
} else {
bool need_slow_path = !k->is_loaded() ||
((int) k->super_check_offset() == in_bytes(Klass::secondary_super_cache_offset()));
- intptr_t super_check_offset = k->is_loaded() ? k->super_check_offset() : -1L;
__ load_klass(klass_RInfo, obj);
// Perform the fast part of the checking logic.
__ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1,
(need_slow_path ? success_target : nullptr),
- failure_target, nullptr,
- RegisterOrConstant(super_check_offset));
+ failure_target, nullptr);
if (need_slow_path) {
// Call out-of-line instance of __ check_klass_subtype_slow_path(...):
address a = Runtime1::entry_for (C1StubId::slow_subtype_check_id);
diff --git a/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp b/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp
index 8b30adb4785..0ada76ccef7 100644
--- a/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp
@@ -557,7 +557,12 @@ OopMapSet* Runtime1::generate_code_for(C1StubId id, StubAssembler* sasm) {
__ z_lg(Rsubklass, 0*BytesPerWord + FrameMap::first_available_sp_in_frame + frame_size, Z_SP);
__ z_lg(Rsuperklass, 1*BytesPerWord + FrameMap::first_available_sp_in_frame + frame_size, Z_SP);
- __ check_klass_subtype_slow_path(Rsubklass, Rsuperklass, Rarray_ptr, Rlength, nullptr, &miss);
+ __ check_klass_subtype_slow_path(Rsubklass,
+ Rsuperklass,
+ Rarray_ptr /* temp_reg */,
+ Rlength /* temp2_reg */,
+ nullptr /* L_success */,
+ &miss /* L_failure */);
// Match falls through here.
i = 0;
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.cpp b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
index 9e1c5cbced3..e8a04e9063b 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
@@ -657,7 +657,7 @@ void MacroAssembler::add2reg(Register r1, int64_t imm, Register r2) {
z_aghik(r1, r2, imm);
return;
}
- z_lgr(r1, r2);
+ lgr_if_needed(r1, r2);
z_aghi(r1, imm);
return;
}
@@ -681,6 +681,37 @@ void MacroAssembler::add2reg(Register r1, int64_t imm, Register r2) {
z_agfi(r1, imm);
}
+void MacroAssembler::add2reg_32(Register r1, int64_t imm, Register r2) {
+ assert(Immediate::is_simm32(imm), "probably an implicit conversion went wrong");
+
+ if (r2 == noreg) { r2 = r1; }
+
+ // Handle special case imm == 0.
+ if (imm == 0) {
+ lr_if_needed(r1, r2);
+ // Nothing else to do.
+ return;
+ }
+
+ if (Immediate::is_simm16(imm)) {
+ if (r1 == r2){
+ z_ahi(r1, imm);
+ return;
+ }
+ if (VM_Version::has_DistinctOpnds()) {
+ z_ahik(r1, r2, imm);
+ return;
+ }
+ lr_if_needed(r1, r2);
+ z_ahi(r1, imm);
+ return;
+ }
+
+ // imm is simm32
+ lr_if_needed(r1, r2);
+ z_afi(r1, imm);
+}
+
// Generic operation r := b + x + d
//
// Addition of several operands with address generation semantics - sort of:
@@ -2981,21 +3012,15 @@ void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
Label* L_success,
Label* L_failure,
Label* L_slow_path,
- RegisterOrConstant super_check_offset) {
+ Register super_check_offset) {
+ // Input registers must not overlap.
+ assert_different_registers(sub_klass, super_klass, temp1_reg, super_check_offset);
- const int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
const int sco_offset = in_bytes(Klass::super_check_offset_offset());
-
- bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
- bool need_slow_path = (must_load_sco ||
- super_check_offset.constant_or_zero() == sc_offset);
+ bool must_load_sco = ! super_check_offset->is_valid();
// Input registers must not overlap.
- assert_different_registers(sub_klass, super_klass, temp1_reg);
- if (super_check_offset.is_register()) {
- assert_different_registers(sub_klass, super_klass,
- super_check_offset.as_register());
- } else if (must_load_sco) {
+ if (must_load_sco) {
assert(temp1_reg != noreg, "supply either a temp or a register offset");
}
@@ -3006,9 +3031,7 @@ void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; }
if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; }
if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; }
- assert(label_nulls <= 1 ||
- (L_slow_path == &L_fallthrough && label_nulls <= 2 && !need_slow_path),
- "at most one null in the batch, usually");
+ assert(label_nulls <= 1 || (L_slow_path == &L_fallthrough && label_nulls <= 2), "at most one null in the batch, usually");
BLOCK_COMMENT("check_klass_subtype_fast_path {");
// If the pointers are equal, we are done (e.g., String[] elements).
@@ -3023,10 +3046,12 @@ void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
// Check the supertype display, which is uint.
if (must_load_sco) {
z_llgf(Rsuper_check_offset, sco_offset, super_klass);
- super_check_offset = RegisterOrConstant(Rsuper_check_offset);
+ super_check_offset = Rsuper_check_offset;
}
+
Address super_check_addr(sub_klass, super_check_offset, 0);
z_cg(super_klass, super_check_addr); // compare w/ displayed supertype
+ branch_optimized(Assembler::bcondEqual, *L_success);
// This check has worked decisively for primary supers.
// Secondary supers are sought in the super_cache ('super_cache_addr').
@@ -3044,46 +3069,27 @@ void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
if (&(label) == &L_fallthrough) { /*do nothing*/ } \
else { branch_optimized(Assembler::bcondAlways, label); } /*omit semicolon*/
- if (super_check_offset.is_register()) {
- branch_optimized(Assembler::bcondEqual, *L_success);
- z_cfi(super_check_offset.as_register(), sc_offset);
- if (L_failure == &L_fallthrough) {
- branch_optimized(Assembler::bcondEqual, *L_slow_path);
- } else {
- branch_optimized(Assembler::bcondNotEqual, *L_failure);
- final_jmp(*L_slow_path);
- }
- } else if (super_check_offset.as_constant() == sc_offset) {
- // Need a slow path; fast failure is impossible.
- if (L_slow_path == &L_fallthrough) {
- branch_optimized(Assembler::bcondEqual, *L_success);
- } else {
- branch_optimized(Assembler::bcondNotEqual, *L_slow_path);
- final_jmp(*L_success);
- }
+ z_cfi(super_check_offset, in_bytes(Klass::secondary_super_cache_offset()));
+ if (L_failure == &L_fallthrough) {
+ branch_optimized(Assembler::bcondEqual, *L_slow_path);
} else {
- // No slow path; it's a fast decision.
- if (L_failure == &L_fallthrough) {
- branch_optimized(Assembler::bcondEqual, *L_success);
- } else {
- branch_optimized(Assembler::bcondNotEqual, *L_failure);
- final_jmp(*L_success);
- }
+ branch_optimized(Assembler::bcondNotEqual, *L_failure);
+ final_jmp(*L_slow_path);
}
bind(L_fallthrough);
-#undef local_brc
#undef final_jmp
BLOCK_COMMENT("} check_klass_subtype_fast_path");
// fallthru (to slow path)
}
-void MacroAssembler::check_klass_subtype_slow_path(Register Rsubklass,
- Register Rsuperklass,
- Register Rarray_ptr, // tmp
- Register Rlength, // tmp
- Label* L_success,
- Label* L_failure) {
+void MacroAssembler::check_klass_subtype_slow_path_linear(Register Rsubklass,
+ Register Rsuperklass,
+ Register Rarray_ptr, // tmp
+ Register Rlength, // tmp
+ Label* L_success,
+ Label* L_failure,
+ bool set_cond_codes /* unused */) {
// Input registers must not overlap.
// Also check for R1 which is explicitly used here.
assert_different_registers(Z_R1, Rsubklass, Rsuperklass, Rarray_ptr, Rlength);
@@ -3106,7 +3112,7 @@ void MacroAssembler::check_klass_subtype_slow_path(Register Rsubklass,
NearLabel loop_iterate, loop_count, match;
- BLOCK_COMMENT("check_klass_subtype_slow_path {");
+ BLOCK_COMMENT("check_klass_subtype_slow_path_linear {");
z_lg(Rarray_ptr, ss_offset, Rsubklass);
load_and_test_int(Rlength, Address(Rarray_ptr, length_offset));
@@ -3134,18 +3140,151 @@ void MacroAssembler::check_klass_subtype_slow_path(Register Rsubklass,
branch_optimized(Assembler::bcondAlways, *L_failure);
// Got a hit. Return success (zero result). Set cache.
- // Cache load doesn't happen here. For speed it is directly emitted by the compiler.
+ // Cache load doesn't happen here. For speed, it is directly emitted by the compiler.
BIND(match);
- z_stg(Rsuperklass, sc_offset, Rsubklass); // Save result to cache.
-
+ if (UseSecondarySupersCache) {
+ z_stg(Rsuperklass, sc_offset, Rsubklass); // Save result to cache.
+ }
final_jmp(*L_success);
// Exit to the surrounding code.
BIND(L_fallthrough);
-#undef local_brc
#undef final_jmp
+ BLOCK_COMMENT("} check_klass_subtype_slow_path_linear");
+}
+
+// If Register r is invalid, remove a new register from
+// available_regs, and add new register to regs_to_push.
+Register MacroAssembler::allocate_if_noreg(Register r,
+ RegSetIterator &available_regs,
+ RegSet ®s_to_push) {
+ if (!r->is_valid()) {
+ r = *available_regs++;
+ regs_to_push += r;
+ }
+ return r;
+}
+
+// check_klass_subtype_slow_path_table() looks for super_klass in the
+// hash table belonging to super_klass, branching to L_success or
+// L_failure as appropriate. This is essentially a shim which
+// allocates registers as necessary and then calls
+// lookup_secondary_supers_table() to do the work. Any of the temp
+// regs may be noreg, in which case this logic will choose some
+// registers push and pop them from the stack.
+void MacroAssembler::check_klass_subtype_slow_path_table(Register sub_klass,
+ Register super_klass,
+ Register temp_reg,
+ Register temp2_reg,
+ Register temp3_reg,
+ Register temp4_reg,
+ Register result_reg,
+ Label* L_success,
+ Label* L_failure,
+ bool set_cond_codes) {
+ BLOCK_COMMENT("check_klass_subtype_slow_path_table {");
+
+ RegSet temps = RegSet::of(temp_reg, temp2_reg, temp3_reg, temp4_reg);
+
+ assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, temp4_reg);
+
+ Label L_fallthrough;
+ int label_nulls = 0;
+ if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; }
+ if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; }
+ assert(label_nulls <= 1, "at most one null in the batch");
+
+ RegSetIterator available_regs
+ // Z_R0 will be used to hold Z_R15(Z_SP) while pushing a new frame, So don't use that here.
+ // Z_R1 will be used to hold r_bitmap in lookup_secondary_supers_table_var, so can't be used
+ // Z_R2, Z_R3, Z_R4 will be used in secondary_supers_verify, for the failure reporting
+ = (RegSet::range(Z_R0, Z_R15) - temps - sub_klass - super_klass - Z_R1_scratch - Z_R0_scratch - Z_R2 - Z_R3 - Z_R4).begin();
+
+ RegSet pushed_regs;
+
+ temp_reg = allocate_if_noreg(temp_reg, available_regs, pushed_regs);
+ temp2_reg = allocate_if_noreg(temp2_reg, available_regs, pushed_regs);
+ temp3_reg = allocate_if_noreg(temp3_reg, available_regs, pushed_regs);;
+ temp4_reg = allocate_if_noreg(temp4_reg, available_regs, pushed_regs);
+ result_reg = allocate_if_noreg(result_reg, available_regs, pushed_regs);
+
+ const int frame_size = pushed_regs.size() * BytesPerWord + frame::z_abi_160_size;
+
+ // Push & save registers
+ {
+ int i = 0;
+ save_return_pc();
+ push_frame(frame_size);
+
+ for (auto it = pushed_regs.begin(); *it != noreg; i++) {
+ z_stg(*it++, i * BytesPerWord + frame::z_abi_160_size, Z_SP);
+ }
+ assert(i * BytesPerWord + frame::z_abi_160_size == frame_size, "sanity");
+ }
+
+ lookup_secondary_supers_table_var(sub_klass,
+ super_klass,
+ temp_reg, temp2_reg, temp3_reg, temp4_reg, result_reg);
+
+ // NOTE: Condition Code should not be altered before jump instruction below !!!!
+ z_cghi(result_reg, 0);
+
+ {
+ int i = 0;
+ for (auto it = pushed_regs.begin(); *it != noreg; ++i) {
+ z_lg(*it++, i * BytesPerWord + frame::z_abi_160_size, Z_SP);
+ }
+ assert(i * BytesPerWord + frame::z_abi_160_size == frame_size, "sanity");
+ pop_frame();
+ restore_return_pc();
+ }
+
+ // NB! Callers may assume that, when set_cond_codes is true, this
+ // code sets temp2_reg to a nonzero value.
+ if (set_cond_codes) {
+ z_lghi(temp2_reg, 1);
+ }
+
+ branch_optimized(bcondNotEqual, *L_failure);
+
+ if(L_success != &L_fallthrough) {
+ z_bru(*L_success);
+ }
+
+ bind(L_fallthrough);
+ BLOCK_COMMENT("} check_klass_subtype_slow_path_table");
+}
+
+void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
+ Register super_klass,
+ Register temp_reg,
+ Register temp2_reg,
+ Label* L_success,
+ Label* L_failure,
+ bool set_cond_codes) {
+ BLOCK_COMMENT("check_klass_subtype_slow_path {");
+ if (UseSecondarySupersTable) {
+ check_klass_subtype_slow_path_table(sub_klass,
+ super_klass,
+ temp_reg,
+ temp2_reg,
+ /*temp3*/noreg,
+ /*temp4*/noreg,
+ /*result*/noreg,
+ L_success,
+ L_failure,
+ set_cond_codes);
+ } else {
+ check_klass_subtype_slow_path_linear(sub_klass,
+ super_klass,
+ temp_reg,
+ temp2_reg,
+ L_success,
+ L_failure,
+ set_cond_codes);
+ }
BLOCK_COMMENT("} check_klass_subtype_slow_path");
}
@@ -3206,17 +3345,17 @@ do { \
} while(0)
// Note: this method also kills Z_R1_scratch register on machines older than z15
-void MacroAssembler::lookup_secondary_supers_table(Register r_sub_klass,
- Register r_super_klass,
- Register r_temp1,
- Register r_temp2,
- Register r_temp3,
- Register r_temp4,
- Register r_result,
- u1 super_klass_slot) {
+void MacroAssembler::lookup_secondary_supers_table_const(Register r_sub_klass,
+ Register r_super_klass,
+ Register r_temp1,
+ Register r_temp2,
+ Register r_temp3,
+ Register r_temp4,
+ Register r_result,
+ u1 super_klass_slot) {
NearLabel L_done, L_failure;
- BLOCK_COMMENT("lookup_secondary_supers_table {");
+ BLOCK_COMMENT("lookup_secondary_supers_table_const {");
const Register
r_array_base = r_temp1,
@@ -3291,7 +3430,7 @@ void MacroAssembler::lookup_secondary_supers_table(Register r_sub_klass,
z_lghi(r_result, 1);
bind(L_done);
- BLOCK_COMMENT("} lookup_secondary_supers_table");
+ BLOCK_COMMENT("} lookup_secondary_supers_table_const");
if (VerifySecondarySupers) {
verify_secondary_supers_table(r_sub_klass, r_super_klass, r_result,
@@ -3299,6 +3438,116 @@ void MacroAssembler::lookup_secondary_supers_table(Register r_sub_klass,
}
}
+// At runtime, return 0 in result if r_super_klass is a superclass of
+// r_sub_klass, otherwise return nonzero. Use this version of
+// lookup_secondary_supers_table() if you don't know ahead of time
+// which superclass will be searched for. Used by interpreter and
+// runtime stubs. It is larger and has somewhat greater latency than
+// the version above, which takes a constant super_klass_slot.
+void MacroAssembler::lookup_secondary_supers_table_var(Register r_sub_klass,
+ Register r_super_klass,
+ Register temp1,
+ Register temp2,
+ Register temp3,
+ Register temp4,
+ Register result) {
+ assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result, Z_R1_scratch);
+
+ Label L_done, L_failure;
+
+ BLOCK_COMMENT("lookup_secondary_supers_table_var {");
+
+ const Register
+ r_array_index = temp3,
+ slot = temp4, // NOTE: "slot" can't be Z_R0 otherwise z_sllg and z_rllg instructions below will mess up!!!!
+ r_bitmap = Z_R1_scratch;
+
+ z_llgc(slot, Address(r_super_klass, Klass::hash_slot_offset()));
+
+ // Initialize r_result with 0 (indicating success). If searching fails, r_result will be loaded
+ // with 1 (failure) at the end of this method.
+ clear_reg(result, true /* whole_reg */, false /* set_cc */); // result = 0
+
+ z_lg(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset()));
+
+ // First check the bitmap to see if super_klass might be present. If
+ // the bit is zero, we are certain that super_klass is not one of
+ // the secondary supers.
+ z_xilf(slot, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 1)); // slot ^ 63 === 63 - slot (mod 64)
+ z_sllg(r_array_index, r_bitmap, /*d2 = */ 0, /* b2 = */ slot);
+
+ testbit(r_array_index, Klass::SECONDARY_SUPERS_TABLE_SIZE - 1);
+ branch_optimized(bcondAllZero, L_failure);
+
+ const Register
+ r_array_base = temp1,
+ r_array_length = temp2;
+
+ // Get the first array index that can contain super_klass into r_array_index.
+ // NOTE: Z_R1_scratch is holding bitmap (look above for r_bitmap). So let's try to save it.
+ // On the other hand, r_array_base/temp1 is free at current moment (look at the load operation below).
+ pop_count_long(r_array_index, r_array_index, temp1); // kills r_array_base/temp1 on machines older than z15
+
+ // The value i in r_array_index is >= 1, so even though r_array_base
+ // points to the length, we don't need to adjust it to point to the data.
+ assert(Array::base_offset_in_bytes() == wordSize, "Adjust this code");
+ assert(Array::length_offset_in_bytes() == 0, "Adjust this code");
+
+ // We will consult the secondary-super array.
+ z_lg(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
+
+ // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word.
+ z_sllg(r_array_index, r_array_index, LogBytesPerWord); // scale, r_array_index is loaded by popcnt above
+
+ z_cg(r_super_klass, Address(r_array_base, r_array_index));
+ branch_optimized(bcondEqual, L_done); // found a match
+
+ // Note: this is a small hack:
+ //
+ // The operation "(slot ^ 63) === 63 - slot (mod 64)" has already been performed above.
+ // Since we lack a rotate-right instruction, we achieve the same effect by rotating left
+ // by "64 - slot" positions. This produces the result equivalent to a right rotation by "slot" positions.
+ //
+ // => initial slot value
+ // => slot = 63 - slot // done above with that z_xilf instruction
+ // => slot = 64 - slot // need to do for rotating right by "slot" positions
+ // => slot = 64 - (63 - slot)
+ // => slot = slot - 63 + 64
+ // => slot = slot + 1
+ //
+ // So instead of rotating-left by 64-slot times, we can, for now, just rotate left by slot+1 and it would be fine.
+
+ // Linear probe. Rotate the bitmap so that the next bit to test is
+ // in Bit 1.
+ z_aghi(slot, 1); // slot = slot + 1
+
+ z_rllg(r_bitmap, r_bitmap, /*d2=*/ 0, /*b2=*/ slot);
+ testbit(r_bitmap, 1);
+ branch_optimized(bcondAllZero, L_failure);
+
+ // The slot we just inspected is at secondary_supers[r_array_index - 1].
+ // The next slot to be inspected, by the logic we're about to call,
+ // is secondary_supers[r_array_index]. Bits 0 and 1 in the bitmap
+ // have been checked.
+ lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index,
+ r_bitmap, /*temp=*/ r_array_length, result, /*is_stub*/false);
+
+ // pass whatever we got from slow path
+ z_bru(L_done);
+
+ bind(L_failure);
+ z_lghi(result, 1); // load 1 to represent failure
+
+ bind(L_done);
+
+ BLOCK_COMMENT("} lookup_secondary_supers_table_var");
+
+ if (VerifySecondarySupers) {
+ verify_secondary_supers_table(r_sub_klass, r_super_klass, result,
+ temp1, temp2, temp3);
+ }
+}
+
// Called by code generated by check_klass_subtype_slow_path
// above. This is called when there is a collision in the hashed
// lookup in the secondary supers array.
@@ -3306,15 +3555,18 @@ void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_kl
Register r_array_base,
Register r_array_index,
Register r_bitmap,
+ Register r_temp,
Register r_result,
- Register r_temp1) {
- assert_different_registers(r_super_klass, r_array_base, r_array_index, r_bitmap, r_result, r_temp1);
+ bool is_stub) {
+ assert_different_registers(r_super_klass, r_array_base, r_array_index, r_bitmap, r_result, r_temp);
const Register
- r_array_length = r_temp1,
+ r_array_length = r_temp,
r_sub_klass = noreg;
- LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS;
+ if(is_stub) {
+ LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS;
+ }
BLOCK_COMMENT("lookup_secondary_supers_table_slow_path {");
NearLabel L_done, L_failure;
@@ -3343,8 +3595,10 @@ void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_kl
{ // This is conventional linear probing, but instead of terminating
// when a null entry is found in the table, we maintain a bitmap
// in which a 0 indicates missing entries.
- // The check above guarantees there are 0s in the bitmap, so the loop
- // eventually terminates.
+ // As long as the bitmap is not completely full,
+ // array_length == popcount(bitmap). The array_length check above
+ // guarantees there are 0s in the bitmap, so the loop eventually
+ // terminates.
#ifdef ASSERT
// r_result is set to 0 by lookup_secondary_supers_table.
@@ -3417,8 +3671,6 @@ void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
const Register r_one = Z_R0_scratch;
z_lghi(r_one, 1); // for locgr down there, to a load result for failure
- LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS;
-
BLOCK_COMMENT("verify_secondary_supers_table {");
Label L_passed, L_failure;
@@ -3573,11 +3825,11 @@ void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Regis
Register zero = temp;
Register monitor_tagged = displacedHeader; // Tagged with markWord::monitor_value.
- // Try to CAS owner (no owner => current thread's _lock_id).
+ // Try to CAS owner (no owner => current thread's _monitor_owner_id).
// If csg succeeds then CR=EQ, otherwise, register zero is filled
// with the current owner.
z_lghi(zero, 0);
- z_lg(Z_R0_scratch, Address(Z_thread, JavaThread::lock_id_offset()));
+ z_lg(Z_R0_scratch, Address(Z_thread, JavaThread::monitor_owner_id_offset()));
z_csg(zero, Z_R0_scratch, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), monitor_tagged);
// Store a non-null value into the box.
@@ -3652,7 +3904,7 @@ void MacroAssembler::compiler_fast_unlock_object(Register oop, Register box, Reg
// Handle existing monitor.
bind(object_has_monitor);
- z_lg(Z_R0_scratch, Address(Z_thread, JavaThread::lock_id_offset()));
+ z_lg(Z_R0_scratch, Address(Z_thread, JavaThread::monitor_owner_id_offset()));
z_cg(Z_R0_scratch, Address(currentHeader, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
z_brne(done);
@@ -6356,11 +6608,11 @@ void MacroAssembler::compiler_fast_lock_lightweight_object(Register obj, Registe
const Address recursions_address(tmp1_monitor, ObjectMonitor::recursions_offset() - monitor_tag);
- // Try to CAS owner (no owner => current thread's _lock_id).
+ // Try to CAS owner (no owner => current thread's _monitor_owner_id).
// If csg succeeds then CR=EQ, otherwise, register zero is filled
// with the current owner.
z_lghi(zero, 0);
- z_lg(Z_R0_scratch, Address(Z_thread, JavaThread::lock_id_offset()));
+ z_lg(Z_R0_scratch, Address(Z_thread, JavaThread::monitor_owner_id_offset()));
z_csg(zero, Z_R0_scratch, owner_address);
z_bre(monitor_locked);
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.hpp b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
index 7806fef3ce8..15968812818 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
@@ -156,7 +156,9 @@ class MacroAssembler: public Assembler {
unsigned int mul_reg64_const16(Register rval, Register work, int cval);
// Generic operation r1 := r2 + imm.
- void add2reg(Register r1, int64_t imm, Register r2 = noreg);
+ void add2reg (Register r1, int64_t imm, Register r2 = noreg);
+ void add2reg_32(Register r1, int64_t imm, Register r2 = noreg);
+
// Generic operation r := b + x + d.
void add2reg_with_index(Register r, int64_t d, Register x, Register b = noreg);
@@ -694,7 +696,7 @@ class MacroAssembler: public Assembler {
Label* L_success,
Label* L_failure,
Label* L_slow_path,
- RegisterOrConstant super_check_offset = RegisterOrConstant(-1));
+ Register super_check_offset = noreg);
// The rest of the type check; must be wired to a corresponding fast path.
// It does not repeat the fast path logic, so don't use it standalone.
@@ -706,25 +708,62 @@ class MacroAssembler: public Assembler {
Register Rarray_ptr, // tmp
Register Rlength, // tmp
Label* L_success,
- Label* L_failure);
+ Label* L_failure,
+ bool set_cond_codes = false);
+
+ void check_klass_subtype_slow_path_linear(Register sub_klass,
+ Register super_klass,
+ Register temp_reg,
+ Register temp2_reg,
+ Label* L_success,
+ Label* L_failure,
+ bool set_cond_codes = false);
+
+ void check_klass_subtype_slow_path_table(Register sub_klass,
+ Register super_klass,
+ Register temp_reg,
+ Register temp2_reg,
+ Register temp3_reg,
+ Register temp4_reg,
+ Register result_reg,
+ Label* L_success,
+ Label* L_failure,
+ bool set_cond_codes = false);
+
+ // If r is valid, return r.
+ // If r is invalid, remove a register r2 from available_regs, add r2
+ // to regs_to_push, then return r2.
+ Register allocate_if_noreg(const Register r,
+ RegSetIterator &available_regs,
+ RegSet ®s_to_push);
void repne_scan(Register r_addr, Register r_value, Register r_count, Register r_scratch);
- void lookup_secondary_supers_table(Register r_sub_klass,
- Register r_super_klass,
- Register r_temp1,
- Register r_temp2,
- Register r_temp3,
- Register r_temp4,
- Register r_result,
- u1 super_klass_slot);
+ // Secondary subtype checking
+ void lookup_secondary_supers_table_var(Register sub_klass,
+ Register r_super_klass,
+ Register temp1,
+ Register temp2,
+ Register temp3,
+ Register temp4,
+ Register result);
+
+ void lookup_secondary_supers_table_const(Register r_sub_klass,
+ Register r_super_klass,
+ Register r_temp1,
+ Register r_temp2,
+ Register r_temp3,
+ Register r_temp4,
+ Register r_result,
+ u1 super_klass_slot);
void lookup_secondary_supers_table_slow_path(Register r_super_klass,
Register r_array_base,
Register r_array_index,
Register r_bitmap,
+ Register r_temp,
Register r_result,
- Register r_temp1);
+ bool is_stub);
void verify_secondary_supers_table(Register r_sub_klass,
Register r_super_klass,
diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad
index e1a98139992..0f1d98d54b9 100644
--- a/src/hotspot/cpu/s390/s390.ad
+++ b/src/hotspot/cpu/s390/s390.ad
@@ -9979,8 +9979,9 @@ instruct ShouldNotReachHere() %{
instruct partialSubtypeCheck(rarg1RegP index, rarg2RegP sub, rarg3RegP super, flagsReg pcc,
rarg4RegP scratch1, rarg5RegP scratch2) %{
match(Set index (PartialSubtypeCheck sub super));
+ predicate(!UseSecondarySupersTable);
effect(KILL pcc, KILL scratch1, KILL scratch2);
- ins_cost(10 * DEFAULT_COST);
+ ins_cost(20 * DEFAULT_COST); // slightly larger than the next version
// TODO: s390 port size(FIXED_SIZE);
format %{ " CALL PartialSubtypeCheck\n" %}
ins_encode %{
@@ -9991,21 +9992,45 @@ instruct partialSubtypeCheck(rarg1RegP index, rarg2RegP sub, rarg3RegP super, fl
ins_pipe(pipe_class_dummy);
%}
+// Two versions of partialSubtypeCheck, both used when we need to
+// search for a super class in the secondary supers array. The first
+// is used when we don't know _a priori_ the class being searched
+// for. The second, far more common, is used when we do know: this is
+// used for instanceof, checkcast, and any case where C2 can determine
+// it by constant propagation.
+instruct partialSubtypeCheckVarSuper(rarg2RegP sub, rarg3RegP super,
+ r11TempRegP result,
+ rarg1RegP temp1, rarg4RegP temp2, rarg5RegP temp3, r10TempRegP temp4,
+ flagsReg pcc) %{
+ match(Set result (PartialSubtypeCheck sub super));
+ predicate(UseSecondarySupersTable);
+ effect(KILL pcc, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
+ ins_cost(10 * DEFAULT_COST); // slightly larger than the next version
+ format %{ "partialSubtypeCheck $result, $sub, $super" %}
+ ins_encode %{
+ __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
+ $temp1$$Register, $temp2$$Register, $temp3$$Register, $temp4$$Register,
+ $result$$Register);
+ %}
+ ins_pipe(pipe_class_dummy);
+%}
+
+
instruct partialSubtypeCheckConstSuper(rarg2RegP sub, rarg1RegP super, immP super_con,
r11TempRegP result, rarg5RegP temp1, rarg4RegP temp2,
rarg3RegP temp3, r10TempRegP temp4, flagsReg pcc) %{
match(Set result (PartialSubtypeCheck sub (Binary super super_con)));
predicate(UseSecondarySupersTable);
effect(KILL pcc, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
- ins_cost(7 * DEFAULT_COST); // needs to be less than competing nodes
+ ins_cost(5 * DEFAULT_COST); // smaller than the next version
format %{ "partialSubtypeCheck $result, $sub, $super, $super_con" %}
ins_encode %{
u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
if (InlineSecondarySupersTest) {
- __ lookup_secondary_supers_table($sub$$Register, $super$$Register,
- $temp1$$Register, $temp2$$Register, $temp3$$Register,
- $temp4$$Register, $result$$Register, super_klass_slot);
+ __ lookup_secondary_supers_table_const($sub$$Register, $super$$Register,
+ $temp1$$Register, $temp2$$Register, $temp3$$Register,
+ $temp4$$Register, $result$$Register, super_klass_slot);
} else {
AddressLiteral stub_address(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot));
__ load_const_optimized(Z_ARG4, stub_address);
@@ -10017,21 +10042,6 @@ instruct partialSubtypeCheckConstSuper(rarg2RegP sub, rarg1RegP super, immP supe
ins_pipe(pipe_class_dummy);
%}
-instruct partialSubtypeCheck_vs_zero(flagsReg pcc, rarg2RegP sub, rarg3RegP super, immP0 zero,
- rarg1RegP index, rarg4RegP scratch1, rarg5RegP scratch2) %{
- match(Set pcc (CmpI (PartialSubtypeCheck sub super) zero));
- effect(KILL scratch1, KILL scratch2, KILL index);
- ins_cost(10 * DEFAULT_COST);
- // TODO: s390 port size(FIXED_SIZE);
- format %{ "CALL PartialSubtypeCheck_vs_zero\n" %}
- ins_encode %{
- AddressLiteral stub_address(StubRoutines::zarch::partial_subtype_check());
- __ load_const_optimized(Z_ARG4, stub_address);
- __ z_basr(Z_R14, Z_ARG4);
- %}
- ins_pipe(pipe_class_dummy);
-%}
-
// ============================================================================
// inlined locking and unlocking
diff --git a/src/hotspot/cpu/s390/stubGenerator_s390.cpp b/src/hotspot/cpu/s390/stubGenerator_s390.cpp
index dd9ed4c9546..9e33cd3abe8 100644
--- a/src/hotspot/cpu/s390/stubGenerator_s390.cpp
+++ b/src/hotspot/cpu/s390/stubGenerator_s390.cpp
@@ -635,9 +635,9 @@ class StubGenerator: public StubCodeGenerator {
r_result = Z_R11;
address start = __ pc();
- __ lookup_secondary_supers_table(r_sub_klass, r_super_klass,
- r_array_base, r_array_length, r_array_index,
- r_bitmap, r_result, super_klass_index);
+ __ lookup_secondary_supers_table_const(r_sub_klass, r_super_klass,
+ r_array_base, r_array_length, r_array_index,
+ r_bitmap, r_result, super_klass_index);
__ z_br(Z_R14);
@@ -659,7 +659,7 @@ class StubGenerator: public StubCodeGenerator {
r_result = Z_R11;
__ lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base,
- r_array_index, r_bitmap, r_result, r_temp1);
+ r_array_index, r_bitmap, r_temp1, r_result, /* is_stub */ true);
__ z_br(Z_R14);
diff --git a/src/hotspot/cpu/s390/vm_version_s390.cpp b/src/hotspot/cpu/s390/vm_version_s390.cpp
index ee3f97f26ef..a5195e753bb 100644
--- a/src/hotspot/cpu/s390/vm_version_s390.cpp
+++ b/src/hotspot/cpu/s390/vm_version_s390.cpp
@@ -57,7 +57,7 @@ unsigned int VM_Version::_Icache_lineSize = DEFAULT
// The following list contains the (approximate) announcement/availability
// dates of the many System z generations in existence as of now.
-// Information compiled from https://www.ibm.com/support/techdocs/atsmastr.nsf/WebIndex/TD105503
+// Information compiled from https://www.ibm.com/support/pages/ibm-mainframe-life-cycle-history
// z900: 2000-10
// z990: 2003-06
// z9: 2005-09
@@ -68,12 +68,13 @@ unsigned int VM_Version::_Icache_lineSize = DEFAULT
// z13: 2015-03
// z14: 2017-09
// z15: 2019-09
+// z16: 2022-05
-static const char* z_gen[] = {" ", "G1", "G2", "G3", "G4", "G5", "G6", "G7", "G8", "G9" };
-static const char* z_machine[] = {" ", "2064", "2084", "2094", "2097", "2817", "2827", "2964", "3906", "8561" };
-static const char* z_name[] = {" ", "z900", "z990", "z9 EC", "z10 EC", "z196 EC", "ec12", "z13", "z14", "z15" };
-static const char* z_WDFM[] = {" ", "2006-06-30", "2008-06-30", "2010-06-30", "2012-06-30", "2014-06-30", "2016-12-31", "2019-06-30", "2021-06-30", "tbd" };
-static const char* z_EOS[] = {" ", "2014-12-31", "2014-12-31", "2017-10-31", "2019-12-31", "2021-12-31", "tbd", "tbd", "tbd", "tbd" };
+static const char* z_gen[] = {" ", "G1", "G2", "G3", "G4", "G5", "G6", "G7", "G8", "G9", "G10" };
+static const char* z_machine[] = {" ", "2064", "2084", "2094", "2097", "2817", "2827", "2964", "3906", "8561", "3931" };
+static const char* z_name[] = {" ", "z900", "z990", "z9 EC", "z10 EC", "z196 EC", "ec12", "z13", "z14", "z15", "z16" };
+static const char* z_WDFM[] = {" ", "2006-06-30", "2008-06-30", "2010-06-30", "2012-06-30", "2014-06-30", "2016-12-31", "2019-06-30", "2021-06-30", "2024-12-31", "tbd" };
+static const char* z_EOS[] = {" ", "2014-12-31", "2014-12-31", "2017-10-31", "2019-12-31", "2021-12-31", "2023-12-31", "2024-12-31", "tbd", "tbd", "tbd" };
static const char* z_features[] = {" ",
"system-z, g1-z900, ldisp",
"system-z, g2-z990, ldisp_fast",
@@ -83,7 +84,9 @@ static const char* z_features[] = {" ",
"system-z, g6-ec12, ldisp_fast, extimm, pcrel_load/store, cmpb, cond_load/store, interlocked_update, txm",
"system-z, g7-z13, ldisp_fast, extimm, pcrel_load/store, cmpb, cond_load/store, interlocked_update, txm, vectorinstr",
"system-z, g8-z14, ldisp_fast, extimm, pcrel_load/store, cmpb, cond_load/store, interlocked_update, txm, vectorinstr, instrext2, venh1",
- "system-z, g9-z15, ldisp_fast, extimm, pcrel_load/store, cmpb, cond_load/store, interlocked_update, txm, vectorinstr, instrext2, venh1, instrext3, venh2"
+ "system-z, g9-z15, ldisp_fast, extimm, pcrel_load/store, cmpb, cond_load/store, interlocked_update, txm, vectorinstr, instrext2, venh1, instrext3, venh2",
+ "system-z, g10-z16, ldisp_fast, extimm, pcrel_load/store, cmpb, cond_load/store, interlocked_update, txm, vectorinstr, instrext2, venh1, instrext3, venh2,"
+ "bear_enh, sort_enh, nnpa_assist, storage_key_removal, vpack_decimal_enh"
};
void VM_Version::initialize() {
@@ -337,6 +340,11 @@ int VM_Version::get_model_index() {
// is the index of the oldest detected model.
int ambiguity = 0;
int model_ix = 0;
+ if (is_z16()) {
+ model_ix = 10;
+ ambiguity++;
+ }
+
if (is_z15()) {
model_ix = 9;
ambiguity++;
diff --git a/src/hotspot/cpu/s390/vm_version_s390.hpp b/src/hotspot/cpu/s390/vm_version_s390.hpp
index 31fdedc59f6..49e6f5686f6 100644
--- a/src/hotspot/cpu/s390/vm_version_s390.hpp
+++ b/src/hotspot/cpu/s390/vm_version_s390.hpp
@@ -48,8 +48,8 @@ class VM_Version: public Abstract_VM_Version {
#define StoreFacilityListExtendedMask 0x0100000000000000UL // z9
#define ETF2Mask 0x0000800000000000UL // z900
#define CryptoFacilityMask 0x0000400000000000UL // z990 (aka message-security assist)
-#define LongDispFacilityMask 0x0000200000000000UL // z900 with microcode update
-#define LongDispFacilityHighPerfMask 0x0000300000000000UL // z990
+#define LongDispFacilityMask 0x0000200000000000UL // z900 with microcode update, Bit: 18
+#define LongDispFacilityHighPerfMask 0x0000100000000000UL // z990, Bit: 19
#define HFPMultiplyAndAddMask 0x0000080000000000UL // z990
#define ExtImmedFacilityMask 0x0000040000000000UL // z9
#define ETF3Mask 0x0000020000000000UL // z990/z9 (?)
@@ -64,7 +64,8 @@ class VM_Version: public Abstract_VM_Version {
#define ExecuteExtensionsMask 0x0000000010000000UL // z10
#define FPExtensionsMask 0x0000000004000000UL // z196
#define FPSupportEnhancementsMask 0x0000000000400000UL // z10
-#define DecimalFloatingPointMask 0x0000000000300000UL // z10
+#define DecimalFloatingPointMask 0x0000000000200000UL // z10, Bit: 42
+#define DecimalFloatingPointHighPerfMask 0x0000000000100000UL // z10, Bit: 43
// z196 begin
#define DistinctOpndsMask 0x0000000000040000UL // z196
#define FastBCRSerializationMask DistinctOpndsMask
@@ -114,6 +115,12 @@ class VM_Version: public Abstract_VM_Version {
#define VectorPackedDecimalEnhMask 0x0000008000000000UL // z15
#define CryptoExtension9Mask 0x0000001000000000UL // z15 (aka message-security assist extension 9)
#define DeflateMask 0x0000010000000000UL // z15
+#define NNPAssistFacilityMask 0x0000000004000000UL // z16, Neural-network-processing-assist facility, Bit: 165
+
+// ----------------------------------------------
+// --- FeatureBitString Bits 193..200 (DW[3]) ---
+// ----------------------------------------------
+#define BEAREnhFacilityMask 0x4000000000000000UL // z16, BEAR-enhancement facility, Bit: 193
enum {
_max_cache_levels = 8, // As limited by ECAG instruction.
@@ -179,9 +186,10 @@ class VM_Version: public Abstract_VM_Version {
static bool is_z10() { return has_GnrlInstrExtensions() && !has_DistinctOpnds(); }
static bool is_z196() { return has_DistinctOpnds() && !has_MiscInstrExt(); }
static bool is_ec12() { return has_MiscInstrExt() && !has_CryptoExt5(); }
- static bool is_z13() { return has_CryptoExt5() && !has_MiscInstrExt2();}
- static bool is_z14() { return has_MiscInstrExt2() && !has_MiscInstrExt3();}
- static bool is_z15() { return has_MiscInstrExt3();}
+ static bool is_z13() { return has_CryptoExt5() && !has_MiscInstrExt2(); }
+ static bool is_z14() { return has_MiscInstrExt2() && !has_MiscInstrExt3(); }
+ static bool is_z15() { return has_MiscInstrExt3() && !has_BEAR_Enh_Facility(); }
+ static bool is_z16() { return has_BEAR_Enh_Facility(); }
// Need to use nested class with unscoped enum.
// C++11 declaration "enum class Cipher { ... } is not supported.
@@ -455,6 +463,7 @@ class VM_Version: public Abstract_VM_Version {
static bool has_FPExtensions() { return (_features[0] & FPExtensionsMask) == FPExtensionsMask; }
static bool has_FPSupportEnhancements() { return (_features[0] & FPSupportEnhancementsMask) == FPSupportEnhancementsMask; }
static bool has_DecimalFloatingPoint() { return (_features[0] & DecimalFloatingPointMask) == DecimalFloatingPointMask; }
+ static bool has_DecimalFloatingPointHighPerf() { return (_features[0] & DecimalFloatingPointHighPerfMask) == DecimalFloatingPointHighPerfMask; }
static bool has_InterlockedAccessV1() { return (_features[0] & InterlockedAccess1Mask) == InterlockedAccess1Mask; }
static bool has_LoadAndALUAtomicV1() { return (_features[0] & InterlockedAccess1Mask) == InterlockedAccess1Mask; }
static bool has_PopCount() { return (_features[0] & PopulationCountMask) == PopulationCountMask; }
@@ -486,6 +495,9 @@ class VM_Version: public Abstract_VM_Version {
static bool has_VectorPackedDecimal() { return (_features[2] & VectorPackedDecimalMask) == VectorPackedDecimalMask; }
static bool has_VectorPackedDecimalEnh() { return (_features[2] & VectorPackedDecimalEnhMask) == VectorPackedDecimalEnhMask; }
+ static bool has_BEAR_Enh_Facility() { return (_features[3] & BEAREnhFacilityMask) == BEAREnhFacilityMask; }
+ static bool has_NNP_Assist_Facility() { return (_features[2] & NNPAssistFacilityMask) == NNPAssistFacilityMask; }
+
// Crypto features query functions.
static bool has_Crypto_AES_GCM128() { return has_Crypto() && test_feature_bit(&_cipher_features_KMA[0], Cipher::_AES128, Cipher::_featureBits); }
static bool has_Crypto_AES_GCM192() { return has_Crypto() && test_feature_bit(&_cipher_features_KMA[0], Cipher::_AES192, Cipher::_featureBits); }
@@ -508,6 +520,7 @@ class VM_Version: public Abstract_VM_Version {
// CPU feature setters (to force model-specific behaviour). Test/debugging only.
static void set_has_DecimalFloatingPoint() { _features[0] |= DecimalFloatingPointMask; }
+ static void set_has_DecimalFloatingPointHighPerf() { _features[0] |= DecimalFloatingPointHighPerfMask; }
static void set_has_FPSupportEnhancements() { _features[0] |= FPSupportEnhancementsMask; }
static void set_has_ExecuteExtensions() { _features[0] |= ExecuteExtensionsMask; }
static void set_has_MemWithImmALUOps() { _features[0] |= GnrlInstrExtFacilityMask; }
@@ -558,6 +571,8 @@ class VM_Version: public Abstract_VM_Version {
static void set_has_VectorEnhancements2() { _features[2] |= VectorEnhancements2Mask; }
static void set_has_VectorPackedDecimal() { _features[2] |= VectorPackedDecimalMask; }
static void set_has_VectorPackedDecimalEnh() { _features[2] |= VectorPackedDecimalEnhMask; }
+ static void set_has_BEAR_Enh_Facility() { _features[3] |= BEAREnhFacilityMask;}
+ static void set_has_NNP_Assist_Facility() { _features[2] |= NNPAssistFacilityMask;}
static void reset_has_VectorFacility() { _features[2] &= ~VectorFacilityMask; }
diff --git a/src/hotspot/cpu/x86/assembler_x86.cpp b/src/hotspot/cpu/x86/assembler_x86.cpp
index 36a122d2946..c2fcbcea71e 100644
--- a/src/hotspot/cpu/x86/assembler_x86.cpp
+++ b/src/hotspot/cpu/x86/assembler_x86.cpp
@@ -2968,7 +2968,7 @@ void Assembler::movb(Register dst, Address src) {
}
void Assembler::movddup(XMMRegister dst, XMMRegister src) {
- NOT_LP64(assert(VM_Version::supports_sse3(), ""));
+ assert(VM_Version::supports_sse3(), "");
int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_rex_vex_w_reverted();
@@ -2977,7 +2977,7 @@ void Assembler::movddup(XMMRegister dst, XMMRegister src) {
}
void Assembler::movddup(XMMRegister dst, Address src) {
- NOT_LP64(assert(VM_Version::supports_sse3(), ""));
+ assert(VM_Version::supports_sse3(), "");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit);
@@ -4614,7 +4614,7 @@ void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int
}
void Assembler::packssdw(XMMRegister dst, XMMRegister src) {
- assert(VM_Version::supports_sse2(), "");
+ NOT_LP64(assert(VM_Version::supports_sse2(), "");)
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
emit_int16(0x6B, (0xC0 | encode));
@@ -4820,7 +4820,7 @@ void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
// In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
- assert(VM_Version::supports_sse2(), "");
+ NOT_LP64(assert(VM_Version::supports_sse2(), "");)
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
emit_int16(0x74, (0xC0 | encode));
@@ -4968,7 +4968,7 @@ void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Addre
// In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
- assert(VM_Version::supports_sse2(), "");
+ NOT_LP64(assert(VM_Version::supports_sse2(), "");)
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
emit_int16(0x75, (0xC0 | encode));
@@ -5017,7 +5017,7 @@ void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vect
// In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
- assert(VM_Version::supports_sse2(), "");
+ NOT_LP64(assert(VM_Version::supports_sse2(), "");)
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
emit_int16(0x76, (0xC0 | encode));
@@ -5122,7 +5122,7 @@ void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) {
}
void Assembler::pmovmskb(Register dst, XMMRegister src) {
- assert(VM_Version::supports_sse2(), "");
+ NOT_LP64(assert(VM_Version::supports_sse2(), "");)
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
emit_int16((unsigned char)0xD7, (0xC0 | encode));
@@ -5188,7 +5188,7 @@ void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
}
void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
- assert(VM_Version::supports_sse2(), "");
+ NOT_LP64(assert(VM_Version::supports_sse2(), "");)
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8);
@@ -5274,14 +5274,14 @@ void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8
}
void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
- assert(VM_Version::supports_sse2(), "");
+ NOT_LP64(assert(VM_Version::supports_sse2(), "");)
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true);
emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
}
void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
- assert(VM_Version::supports_sse2(), "");
+ NOT_LP64(assert(VM_Version::supports_sse2(), "");)
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
@@ -8712,7 +8712,7 @@ void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
}
void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
- assert(VM_Version::supports_sse2(), "");
+ NOT_LP64(assert(VM_Version::supports_sse2(), "");)
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
emit_int16((unsigned char)0xF4, (0xC0 | encode));
@@ -8813,7 +8813,7 @@ void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int v
}
void Assembler::pminsw(XMMRegister dst, XMMRegister src) {
- assert(VM_Version::supports_sse2(), "");
+ NOT_LP64(assert(VM_Version::supports_sse2(), "");)
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
emit_int16((unsigned char)0xEA, (0xC0 | encode));
@@ -8892,7 +8892,7 @@ void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int v
}
void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) {
- assert(VM_Version::supports_sse2(), "");
+ NOT_LP64(assert(VM_Version::supports_sse2(), "");)
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
emit_int16((unsigned char)0xEE, (0xC0 | encode));
@@ -12401,7 +12401,7 @@ void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegist
void Assembler::gf2p8affineqb(XMMRegister dst, XMMRegister src, int imm8) {
assert(VM_Version::supports_gfni(), "");
- assert(VM_Version::supports_sse(), "");
+ NOT_LP64(assert(VM_Version::supports_sse(), "");)
InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
@@ -12409,7 +12409,7 @@ void Assembler::gf2p8affineqb(XMMRegister dst, XMMRegister src, int imm8) {
void Assembler::vgf2p8affineqb(XMMRegister dst, XMMRegister src2, XMMRegister src3, int imm8, int vector_len) {
assert(VM_Version::supports_gfni(), "requires GFNI support");
- assert(VM_Version::supports_sse(), "");
+ NOT_LP64(assert(VM_Version::supports_sse(), "");)
InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
index 8d0af29e91d..008d52dcb1c 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
@@ -319,7 +319,7 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
movptr(Address(boxReg, 0), checked_cast(markWord::unused_mark().value()));
// It's inflated and we use scrReg for ObjectMonitor* in this section.
- movptr(boxReg, Address(r15_thread, JavaThread::lock_id_offset()));
+ movptr(boxReg, Address(r15_thread, JavaThread::monitor_owner_id_offset()));
movq(scrReg, tmpReg);
xorq(tmpReg, tmpReg);
lock();
@@ -625,9 +625,9 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Regist
movptr(Address(box, BasicLock::object_monitor_cache_offset_in_bytes()), monitor);
}
- // Try to CAS owner (no owner => current thread's _lock_id).
+ // Try to CAS owner (no owner => current thread's _monitor_owner_id).
xorptr(rax_reg, rax_reg);
- movptr(box, Address(thread, JavaThread::lock_id_offset()));
+ movptr(box, Address(thread, JavaThread::monitor_owner_id_offset()));
lock(); cmpxchgptr(box, owner_address);
jccb(Assembler::equal, monitor_locked);
diff --git a/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp
index 032b33f40e0..eb6da25d1bc 100644
--- a/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -84,6 +85,10 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LI
LIR_Opr result = gen->new_register(T_INT);
__ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result));
+
+ if (ShenandoahCardBarrier) {
+ post_barrier(access, access.resolved_addr(), new_value.result());
+ }
return result;
}
}
@@ -113,6 +118,9 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt
pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
result /* pre_val */);
}
+ if (ShenandoahCardBarrier) {
+ post_barrier(access, access.resolved_addr(), result);
+ }
}
return result;
diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
index a7682fe0c38..a452850b1e8 100644
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -31,6 +32,7 @@
#include "gc/shenandoah/shenandoahRuntime.hpp"
#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/mode/shenandoahMode.hpp"
#include "interpreter/interpreter.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/sharedRuntime.hpp"
@@ -120,6 +122,29 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
if (is_reference_type(type)) {
+ if (ShenandoahCardBarrier) {
+ bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
+ bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
+ bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
+
+ // We need to save the original element count because the array copy stub
+ // will destroy the value and we need it for the card marking barrier.
+#ifdef _LP64
+ if (!checkcast) {
+ if (!obj_int) {
+ // Save count for barrier
+ __ movptr(r11, count);
+ } else if (disjoint) {
+ // Save dst in r11 in the disjoint case
+ __ movq(r11, dst);
+ }
+ }
+#else
+ if (disjoint) {
+ __ mov(rdx, dst); // save 'to'
+ }
+#endif
+ }
if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
#ifdef _LP64
@@ -140,10 +165,10 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
#endif
assert_different_registers(src, dst, count, thread);
- Label done;
+ Label L_done;
// Short-circuit if count == 0.
__ testptr(count, count);
- __ jcc(Assembler::zero, done);
+ __ jcc(Assembler::zero, L_done);
// Avoid runtime call when not active.
Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
@@ -154,7 +179,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
}
__ testb(gc_state, flags);
- __ jcc(Assembler::zero, done);
+ __ jcc(Assembler::zero, L_done);
save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
@@ -174,13 +199,43 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
- __ bind(done);
+ __ bind(L_done);
NOT_LP64(__ pop(thread);)
}
}
}
+void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+ Register src, Register dst, Register count) {
+
+ if (ShenandoahCardBarrier && is_reference_type(type)) {
+ bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
+ bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
+ bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
+ Register tmp = rax;
+
+#ifdef _LP64
+ if (!checkcast) {
+ if (!obj_int) {
+ // Save count for barrier
+ count = r11;
+ } else if (disjoint) {
+ // Use the saved dst in the disjoint case
+ dst = r11;
+ }
+ } else {
+ tmp = rscratch1;
+ }
+#else
+ if (disjoint) {
+ __ mov(dst, rdx); // restore 'to'
+ }
+#endif
+ gen_write_ref_array_post_barrier(masm, decorators, dst, count, tmp);
+ }
+}
+
void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
Register obj,
Register pre_val,
@@ -555,6 +610,49 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
}
}
+void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
+ assert(ShenandoahCardBarrier, "Should have been checked by caller");
+
+ // Does a store check for the oop in register obj. The content of
+ // register obj is destroyed afterwards.
+
+ ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
+ CardTable* ct = ctbs->card_table();
+
+ __ shrptr(obj, CardTable::card_shift());
+
+ Address card_addr;
+
+ // The calculation for byte_map_base is as follows:
+ // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
+ // So this essentially converts an address to a displacement and it will
+ // never need to be relocated. On 64-bit however the value may be too
+ // large for a 32-bit displacement.
+ intptr_t byte_map_base = (intptr_t)ct->byte_map_base();
+ if (__ is_simm32(byte_map_base)) {
+ card_addr = Address(noreg, obj, Address::times_1, byte_map_base);
+ } else {
+ // By doing it as an ExternalAddress 'byte_map_base' could be converted to a rip-relative
+ // displacement and done in a single instruction given favorable mapping and a
+ // smarter version of as_Address. However, 'ExternalAddress' generates a relocation
+ // entry and that entry is not properly handled by the relocation code.
+ AddressLiteral cardtable((address)byte_map_base, relocInfo::none);
+ Address index(noreg, obj, Address::times_1);
+ card_addr = __ as_Address(ArrayAddress(cardtable, index), rscratch1);
+ }
+
+ int dirty = CardTable::dirty_card_val();
+ if (UseCondCardMark) {
+ Label L_already_dirty;
+ __ cmpb(card_addr, dirty);
+ __ jccb(Assembler::equal, L_already_dirty);
+ __ movb(card_addr, dirty);
+ __ bind(L_already_dirty);
+ } else {
+ __ movb(card_addr, dirty);
+ }
+}
+
void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
@@ -592,7 +690,13 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet
val != noreg /* tosca_live */,
false /* expand_call */);
}
+
BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
+ if (val != noreg) {
+ if (ShenandoahCardBarrier) {
+ store_check(masm, tmp1);
+ }
+ }
NOT_LP64(imasm->restore_bcp());
} else {
BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
@@ -787,6 +891,63 @@ void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
}
}
+#ifdef PRODUCT
+#define BLOCK_COMMENT(str) /* nothing */
+#else
+#define BLOCK_COMMENT(str) __ block_comment(str)
+#endif
+
+#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
+
+#define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
+
+void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+ Register addr, Register count,
+ Register tmp) {
+ assert(ShenandoahCardBarrier, "Should have been checked by caller");
+
+ ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
+ CardTable* ct = bs->card_table();
+ intptr_t disp = (intptr_t) ct->byte_map_base();
+
+ Label L_loop, L_done;
+ const Register end = count;
+ assert_different_registers(addr, end);
+
+ // Zero count? Nothing to do.
+ __ testl(count, count);
+ __ jccb(Assembler::zero, L_done);
+
+#ifdef _LP64
+ __ leaq(end, Address(addr, count, TIMES_OOP, 0)); // end == addr+count*oop_size
+ __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
+ __ shrptr(addr, CardTable::card_shift());
+ __ shrptr(end, CardTable::card_shift());
+ __ subptr(end, addr); // end --> cards count
+
+ __ mov64(tmp, disp);
+ __ addptr(addr, tmp);
+
+ __ BIND(L_loop);
+ __ movb(Address(addr, count, Address::times_1), 0);
+ __ decrement(count);
+ __ jccb(Assembler::greaterEqual, L_loop);
+#else
+ __ lea(end, Address(addr, count, Address::times_ptr, -wordSize));
+ __ shrptr(addr, CardTable::card_shift());
+ __ shrptr(end, CardTable::card_shift());
+ __ subptr(end, addr); // end --> count
+
+ __ BIND(L_loop);
+ Address cardtable(addr, count, Address::times_1, disp);
+ __ movb(cardtable, 0);
+ __ decrement(count);
+ __ jccb(Assembler::greaterEqual, L_loop);
+#endif
+
+ __ BIND(L_done);
+}
+
#undef __
#ifdef COMPILER1
diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
index c8140f4a76a..ae0ad3533e1 100644
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -56,6 +57,12 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
bool tosca_live,
bool expand_call);
+ void store_check(MacroAssembler* masm, Register obj);
+
+ void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+ Register addr, Register count,
+ Register tmp);
+
public:
#ifdef COMPILER1
void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
@@ -71,6 +78,8 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
bool exchange, Register tmp1, Register tmp2);
virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register src, Register dst, Register count);
+ virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+ Register src, Register dst, Register count);
virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp_thread);
virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
index c08e535ee0e..a798dea08cc 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
@@ -5445,7 +5445,6 @@ void MacroAssembler::vallones(XMMRegister dst, int vector_len) {
} else if (VM_Version::supports_avx()) {
vpcmpeqd(dst, dst, dst, vector_len);
} else {
- assert(VM_Version::supports_sse2(), "");
pcmpeqd(dst, dst);
}
}
diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad
index 550c8047034..4667922505c 100644
--- a/src/hotspot/cpu/x86/x86_64.ad
+++ b/src/hotspot/cpu/x86/x86_64.ad
@@ -4424,21 +4424,21 @@ instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp,
predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
match(Set dst (MaxF a b));
effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
- format %{ "maxF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
+ format %{ "maxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
ins_encode %{
__ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
%}
ins_pipe( pipe_slow );
%}
-instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
+instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
match(Set dst (MaxF a b));
- effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
+ effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
- format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
+ format %{ "maxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
ins_encode %{
- emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
+ emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
false /*min*/, true /*single*/);
%}
ins_pipe( pipe_slow );
@@ -4449,21 +4449,21 @@ instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp,
predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
match(Set dst (MaxD a b));
effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
- format %{ "maxD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
+ format %{ "maxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
ins_encode %{
__ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
%}
ins_pipe( pipe_slow );
%}
-instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
+instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
match(Set dst (MaxD a b));
- effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
+ effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
- format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
+ format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
ins_encode %{
- emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
+ emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
false /*min*/, false /*single*/);
%}
ins_pipe( pipe_slow );
@@ -4474,21 +4474,21 @@ instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp,
predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
match(Set dst (MinF a b));
effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
- format %{ "minF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
+ format %{ "minF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
ins_encode %{
__ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
%}
ins_pipe( pipe_slow );
%}
-instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
+instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
match(Set dst (MinF a b));
- effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
+ effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
- format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
+ format %{ "minF_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
ins_encode %{
- emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
+ emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
true /*min*/, true /*single*/);
%}
ins_pipe( pipe_slow );
@@ -4499,21 +4499,21 @@ instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp,
predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
match(Set dst (MinD a b));
effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
- format %{ "minD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
+ format %{ "minD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
ins_encode %{
__ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
%}
ins_pipe( pipe_slow );
%}
-instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
+instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
match(Set dst (MinD a b));
- effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
+ effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
- format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
+ format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
ins_encode %{
- emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
+ emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
true /*min*/, false /*single*/);
%}
ins_pipe( pipe_slow );
diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp
index 44b9571e102..26627c2f8fb 100644
--- a/src/hotspot/os/aix/os_aix.cpp
+++ b/src/hotspot/os/aix/os_aix.cpp
@@ -856,9 +856,9 @@ void os::pd_start_thread(Thread* thread) {
void os::free_thread(OSThread* osthread) {
assert(osthread != nullptr, "osthread not set");
- // We are told to free resources of the argument thread,
- // but we can only really operate on the current thread.
- assert(Thread::current()->osthread() == osthread,
+ // We are told to free resources of the argument thread, but we can only really operate
+ // on the current thread. The current thread may be already detached at this point.
+ assert(Thread::current_or_null() == nullptr || Thread::current()->osthread() == osthread,
"os::free_thread but not current thread");
// Restore caller's signal mask
diff --git a/src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.cpp b/src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.cpp
index 2e56c092a79..16835c83039 100644
--- a/src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.cpp
+++ b/src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -79,7 +79,7 @@ ZPhysicalMemoryBacking::ZPhysicalMemoryBacking(size_t max_capacity)
_initialized(false) {
// Reserve address space for backing memory
- _base = (uintptr_t)os::reserve_memory(max_capacity);
+ _base = (uintptr_t)os::reserve_memory(max_capacity, false, mtJavaHeap);
if (_base == 0) {
// Failed
ZInitialize::error("Failed to reserve address space for backing memory");
diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp
index a1887aa5975..5db846275d4 100644
--- a/src/hotspot/os/bsd/os_bsd.cpp
+++ b/src/hotspot/os/bsd/os_bsd.cpp
@@ -229,8 +229,6 @@ size_t os::rss() {
// Cpu architecture string
#if defined(ZERO)
static char cpu_arch[] = ZERO_LIBARCH;
-#elif defined(IA64)
-static char cpu_arch[] = "ia64";
#elif defined(IA32)
static char cpu_arch[] = "i386";
#elif defined(AMD64)
@@ -765,9 +763,9 @@ void os::pd_start_thread(Thread* thread) {
void os::free_thread(OSThread* osthread) {
assert(osthread != nullptr, "osthread not set");
- // We are told to free resources of the argument thread,
- // but we can only really operate on the current thread.
- assert(Thread::current()->osthread() == osthread,
+ // We are told to free resources of the argument thread, but we can only really operate
+ // on the current thread. The current thread may be already detached at this point.
+ assert(Thread::current_or_null() == nullptr || Thread::current()->osthread() == osthread,
"os::free_thread but not current thread");
// Restore caller's signal mask
@@ -1192,8 +1190,6 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
static Elf32_Half running_arch_code=EM_386;
#elif (defined AMD64)
static Elf32_Half running_arch_code=EM_X86_64;
- #elif (defined IA64)
- static Elf32_Half running_arch_code=EM_IA_64;
#elif (defined __powerpc64__)
static Elf32_Half running_arch_code=EM_PPC64;
#elif (defined __powerpc__)
@@ -1214,7 +1210,7 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
static Elf32_Half running_arch_code=EM_68K;
#else
#error Method os::dll_load requires that one of following is defined:\
- IA32, AMD64, IA64, __powerpc__, ARM, S390, ALPHA, MIPS, MIPSEL, PARISC, M68K
+ IA32, AMD64, __powerpc__, ARM, S390, ALPHA, MIPS, MIPSEL, PARISC, M68K
#endif
// Identify compatibility class for VM's architecture and library's architecture
diff --git a/src/hotspot/os/linux/hugepages.cpp b/src/hotspot/os/linux/hugepages.cpp
index b71593487cf..c04ff7a4ca0 100644
--- a/src/hotspot/os/linux/hugepages.cpp
+++ b/src/hotspot/os/linux/hugepages.cpp
@@ -55,8 +55,7 @@ static size_t scan_default_hugepagesize() {
// large_page_size on Linux is used to round up heap size. x86 uses either
// 2M or 4M page, depending on whether PAE (Physical Address Extensions)
- // mode is enabled. AMD64/EM64T uses 2M page in 64bit mode. IA64 can use
- // page as large as 1G.
+ // mode is enabled. AMD64/EM64T uses 2M page in 64bit mode.
//
// Here we try to figure out page size by parsing /proc/meminfo and looking
// for a line with the following format:
diff --git a/src/hotspot/os/linux/mallocInfoDcmd.hpp b/src/hotspot/os/linux/mallocInfoDcmd.hpp
index 9f52d83fba3..f2559fa1571 100644
--- a/src/hotspot/os/linux/mallocInfoDcmd.hpp
+++ b/src/hotspot/os/linux/mallocInfoDcmd.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -41,10 +41,6 @@ class MallocInfoDcmd : public DCmd {
static const char* impact() {
return "Low";
}
- static const JavaPermission permission() {
- JavaPermission p = { "java.lang.management.ManagementPermission", "monitor", nullptr };
- return p;
- }
void execute(DCmdSource source, TRAPS) override;
};
diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp
index 19714998891..03c06b7f131 100644
--- a/src/hotspot/os/linux/os_linux.cpp
+++ b/src/hotspot/os/linux/os_linux.cpp
@@ -461,26 +461,17 @@ bool os::Linux::get_tick_information(CPUPerfTicks* pticks, int which_logical_cpu
}
#ifndef SYS_gettid
-// i386: 224, ia64: 1105, amd64: 186, sparc: 143
- #ifdef __ia64__
- #define SYS_gettid 1105
+// i386: 224, amd64: 186, sparc: 143
+ #if defined(__i386__)
+ #define SYS_gettid 224
+ #elif defined(__amd64__)
+ #define SYS_gettid 186
+ #elif defined(__sparc__)
+ #define SYS_gettid 143
#else
- #ifdef __i386__
- #define SYS_gettid 224
- #else
- #ifdef __amd64__
- #define SYS_gettid 186
- #else
- #ifdef __sparc__
- #define SYS_gettid 143
- #else
- #error define gettid for the arch
- #endif
- #endif
- #endif
+ #error "Define SYS_gettid for this architecture"
#endif
-#endif
-
+#endif // SYS_gettid
// pid_t gettid()
//
@@ -571,6 +562,7 @@ void os::init_system_properties_values() {
// Base path of extensions installed on the system.
#define SYS_EXT_DIR "/usr/java/packages"
#define EXTENSIONS_DIR "/lib/ext"
+#define JVM_LIB_NAME "libjvm.so"
// Buffer that fits several snprintfs.
// Note that the space for the colon and the trailing null are provided
@@ -585,22 +577,32 @@ void os::init_system_properties_values() {
char *pslash;
os::jvm_path(buf, bufsize);
- // Found the full path to libjvm.so.
- // Now cut the path to /jre if we can.
- pslash = strrchr(buf, '/');
- if (pslash != nullptr) {
- *pslash = '\0'; // Get rid of /libjvm.so.
- }
+ // Found the full path to the binary. It is normally of this structure:
+ // /lib//libjvm.so
+ // but can also be like this for a statically linked binary:
+ // /bin/
pslash = strrchr(buf, '/');
if (pslash != nullptr) {
- *pslash = '\0'; // Get rid of /{client|server|hotspot}.
+ if (strncmp(pslash + 1, JVM_LIB_NAME, strlen(JVM_LIB_NAME)) == 0) {
+ // Binary name is libjvm.so. Get rid of /libjvm.so.
+ *pslash = '\0';
+ }
+
+ // Get rid of /, if binary is libjvm.so,
+ // or cut off /, if it is a statically linked binary.
+ pslash = strrchr(buf, '/');
+ if (pslash != nullptr) {
+ *pslash = '\0';
+ }
}
Arguments::set_dll_dir(buf);
+ // Get rid of /lib, if binary is libjvm.so,
+ // or cut off /bin, if it is a statically linked binary.
if (pslash != nullptr) {
pslash = strrchr(buf, '/');
if (pslash != nullptr) {
- *pslash = '\0'; // Get rid of /lib.
+ *pslash = '\0';
}
}
Arguments::set_java_home(buf);
@@ -1188,9 +1190,9 @@ void os::pd_start_thread(Thread* thread) {
void os::free_thread(OSThread* osthread) {
assert(osthread != nullptr, "osthread not set");
- // We are told to free resources of the argument thread,
- // but we can only really operate on the current thread.
- assert(Thread::current()->osthread() == osthread,
+ // We are told to free resources of the argument thread, but we can only really operate
+ // on the current thread. The current thread may be already detached at this point.
+ assert(Thread::current_or_null() == nullptr || Thread::current()->osthread() == osthread,
"os::free_thread but not current thread");
#ifdef ASSERT
@@ -1778,8 +1780,6 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
static Elf32_Half running_arch_code=EM_386;
#elif (defined AMD64) || (defined X32)
static Elf32_Half running_arch_code=EM_X86_64;
-#elif (defined IA64)
- static Elf32_Half running_arch_code=EM_IA_64;
#elif (defined __sparc) && (defined _LP64)
static Elf32_Half running_arch_code=EM_SPARCV9;
#elif (defined __sparc) && (!defined _LP64)
@@ -1812,7 +1812,7 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
static Elf32_Half running_arch_code=EM_LOONGARCH;
#else
#error Method os::dll_load requires that one of following is defined:\
- AARCH64, ALPHA, ARM, AMD64, IA32, IA64, LOONGARCH64, M68K, MIPS, MIPSEL, PARISC, __powerpc__, __powerpc64__, RISCV, S390, SH, __sparc
+ AARCH64, ALPHA, ARM, AMD64, IA32, LOONGARCH64, M68K, MIPS, MIPSEL, PARISC, __powerpc__, __powerpc64__, RISCV, S390, SH, __sparc
#endif
// Identify compatibility class for VM's architecture and library's architecture
@@ -2803,8 +2803,6 @@ void os::get_summary_cpu_info(char* cpuinfo, size_t length) {
strncpy(cpuinfo, "ARM", length);
#elif defined(IA32)
strncpy(cpuinfo, "x86_32", length);
-#elif defined(IA64)
- strncpy(cpuinfo, "IA64", length);
#elif defined(PPC)
strncpy(cpuinfo, "PPC64", length);
#elif defined(RISCV)
@@ -4668,7 +4666,7 @@ static void workaround_expand_exec_shield_cs_limit() {
*/
char* hint = (char*)(os::Linux::initial_thread_stack_bottom() -
(StackOverflow::stack_guard_zone_size() + page_size));
- char* codebuf = os::attempt_reserve_memory_at(hint, page_size);
+ char* codebuf = os::attempt_reserve_memory_at(hint, page_size, false, mtThread);
if (codebuf == nullptr) {
// JDK-8197429: There may be a stack gap of one megabyte between
@@ -4676,15 +4674,13 @@ static void workaround_expand_exec_shield_cs_limit() {
// Linux kernel workaround for CVE-2017-1000364. If we failed to
// map our codebuf, try again at an address one megabyte lower.
hint -= 1 * M;
- codebuf = os::attempt_reserve_memory_at(hint, page_size);
+ codebuf = os::attempt_reserve_memory_at(hint, page_size, false, mtThread);
}
if ((codebuf == nullptr) || (!os::commit_memory(codebuf, page_size, true))) {
return; // No matter, we tried, best effort.
}
- MemTracker::record_virtual_memory_tag((address)codebuf, mtInternal);
-
log_info(os)("[CS limit NX emulation work-around, exec code at: %p]", codebuf);
// Some code to exec: the 'ret' instruction
diff --git a/src/hotspot/os/linux/trimCHeapDCmd.hpp b/src/hotspot/os/linux/trimCHeapDCmd.hpp
index 32a49798ea8..53b94f4bb56 100644
--- a/src/hotspot/os/linux/trimCHeapDCmd.hpp
+++ b/src/hotspot/os/linux/trimCHeapDCmd.hpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2021 SAP SE. All rights reserved.
- * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -42,10 +42,6 @@ class TrimCLibcHeapDCmd : public DCmd {
static const char* impact() {
return "Low";
}
- static const JavaPermission permission() {
- JavaPermission p = { "java.lang.management.ManagementPermission", "control", nullptr };
- return p;
- }
virtual void execute(DCmdSource source, TRAPS);
};
diff --git a/src/hotspot/os/posix/malloctrace/mallocTracePosix.hpp b/src/hotspot/os/posix/malloctrace/mallocTracePosix.hpp
index f84c9fd1584..8dc2c96cddf 100644
--- a/src/hotspot/os/posix/malloctrace/mallocTracePosix.hpp
+++ b/src/hotspot/os/posix/malloctrace/mallocTracePosix.hpp
@@ -131,11 +131,6 @@ class MallocTraceEnableDCmd : public DCmdWithParser {
return "High";
}
- static const JavaPermission permission() {
- JavaPermission p = { "java.lang.management.ManagementPermission", "control", nullptr };
- return p;
- }
-
virtual void execute(DCmdSource source, TRAPS);
};
@@ -160,11 +155,6 @@ class MallocTraceDisableDCmd : public DCmdWithParser {
return "Low";
}
- static const JavaPermission permission() {
- JavaPermission p = { "java.lang.management.ManagementPermission", "control", nullptr };
- return p;
- }
-
virtual void execute(DCmdSource source, TRAPS);
};
@@ -197,11 +187,6 @@ class MallocTraceDumpDCmd : public DCmdWithParser {
return "Low";
}
- static const JavaPermission permission() {
- JavaPermission p = { "java.lang.management.ManagementPermission", "control", nullptr };
- return p;
- }
-
virtual void execute(DCmdSource source, TRAPS);
};
diff --git a/src/hotspot/os/posix/os_posix.cpp b/src/hotspot/os/posix/os_posix.cpp
index 7d418d4ad43..61214a42969 100644
--- a/src/hotspot/os/posix/os_posix.cpp
+++ b/src/hotspot/os/posix/os_posix.cpp
@@ -854,6 +854,12 @@ void os::dll_unload(void *lib) {
LINUX_ONLY(os::free(l_pathdup));
}
+void* os::lookup_function(const char* name) {
+ // This returns the global symbol in the main executable and its dependencies,
+ // as well as shared objects dynamically loaded with RTLD_GLOBAL flag.
+ return dlsym(RTLD_DEFAULT, name);
+}
+
jlong os::lseek(int fd, jlong offset, int whence) {
return (jlong) ::lseek(fd, offset, whence);
}
@@ -935,51 +941,6 @@ void os::naked_yield() {
sched_yield();
}
-// Builds a platform dependent Agent_OnLoad_ function name
-// which is used to find statically linked in agents.
-// Parameters:
-// sym_name: Symbol in library we are looking for
-// lib_name: Name of library to look in, null for shared libs.
-// is_absolute_path == true if lib_name is absolute path to agent
-// such as "/a/b/libL.so"
-// == false if only the base name of the library is passed in
-// such as "L"
-char* os::build_agent_function_name(const char *sym_name, const char *lib_name,
- bool is_absolute_path) {
- char *agent_entry_name;
- size_t len;
- size_t name_len;
- size_t prefix_len = strlen(JNI_LIB_PREFIX);
- size_t suffix_len = strlen(JNI_LIB_SUFFIX);
- const char *start;
-
- if (lib_name != nullptr) {
- name_len = strlen(lib_name);
- if (is_absolute_path) {
- // Need to strip path, prefix and suffix
- if ((start = strrchr(lib_name, *os::file_separator())) != nullptr) {
- lib_name = ++start;
- }
- if (strlen(lib_name) <= (prefix_len + suffix_len)) {
- return nullptr;
- }
- lib_name += prefix_len;
- name_len = strlen(lib_name) - suffix_len;
- }
- }
- len = (lib_name != nullptr ? name_len : 0) + strlen(sym_name) + 2;
- agent_entry_name = NEW_C_HEAP_ARRAY_RETURN_NULL(char, len, mtThread);
- if (agent_entry_name == nullptr) {
- return nullptr;
- }
- strcpy(agent_entry_name, sym_name);
- if (lib_name != nullptr) {
- strcat(agent_entry_name, "_");
- strncat(agent_entry_name, lib_name, name_len);
- }
- return agent_entry_name;
-}
-
// Sleep forever; naked call to OS-specific sleep; use with CAUTION
void os::infinite_sleep() {
while (true) { // sleep forever ...
@@ -2225,4 +2186,3 @@ const void* os::get_saved_assert_context(const void** sigInfo) {
*sigInfo = nullptr;
return nullptr;
}
-
diff --git a/src/hotspot/os/posix/signals_posix.cpp b/src/hotspot/os/posix/signals_posix.cpp
index bbf122fabfb..ddc7a05c2ff 100644
--- a/src/hotspot/os/posix/signals_posix.cpp
+++ b/src/hotspot/os/posix/signals_posix.cpp
@@ -960,10 +960,6 @@ static bool get_signal_code_description(const siginfo_t* si, enum_sigcode_desc_t
{ SIGILL, ILL_PRVREG, "ILL_PRVREG", "Privileged register." },
{ SIGILL, ILL_COPROC, "ILL_COPROC", "Coprocessor error." },
{ SIGILL, ILL_BADSTK, "ILL_BADSTK", "Internal stack error." },
-#if defined(IA64) && defined(LINUX)
- { SIGILL, ILL_BADIADDR, "ILL_BADIADDR", "Unimplemented instruction address" },
- { SIGILL, ILL_BREAK, "ILL_BREAK", "Application Break instruction" },
-#endif
{ SIGFPE, FPE_INTDIV, "FPE_INTDIV", "Integer divide by zero." },
{ SIGFPE, FPE_INTOVF, "FPE_INTOVF", "Integer overflow." },
{ SIGFPE, FPE_FLTDIV, "FPE_FLTDIV", "Floating-point divide by zero." },
@@ -977,9 +973,6 @@ static bool get_signal_code_description(const siginfo_t* si, enum_sigcode_desc_t
#if defined(AIX)
// no explanation found what keyerr would be
{ SIGSEGV, SEGV_KEYERR, "SEGV_KEYERR", "key error" },
-#endif
-#if defined(IA64) && !defined(AIX)
- { SIGSEGV, SEGV_PSTKOVF, "SEGV_PSTKOVF", "Paragraph stack overflow" },
#endif
{ SIGBUS, BUS_ADRALN, "BUS_ADRALN", "Invalid address alignment." },
{ SIGBUS, BUS_ADRERR, "BUS_ADRERR", "Nonexistent physical address." },
diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp
index 8bb3789bbad..afd8fe01752 100644
--- a/src/hotspot/os/windows/os_windows.cpp
+++ b/src/hotspot/os/windows/os_windows.cpp
@@ -788,9 +788,9 @@ bool os::create_thread(Thread* thread, ThreadType thr_type,
void os::free_thread(OSThread* osthread) {
assert(osthread != nullptr, "osthread not set");
- // We are told to free resources of the argument thread,
- // but we can only really operate on the current thread.
- assert(Thread::current()->osthread() == osthread,
+ // We are told to free resources of the argument thread, but we can only really operate
+ // on the current thread. The current thread may be already detached at this point.
+ assert(Thread::current_or_null() == nullptr || Thread::current()->osthread() == osthread,
"os::free_thread but not current thread");
CloseHandle(osthread->thread_handle());
@@ -1401,6 +1401,12 @@ void* os::dll_lookup(void *lib, const char *name) {
return ret;
}
+void* os::lookup_function(const char* name) {
+ // This is needed only for static builds which are not supported on Windows
+ ShouldNotReachHere();
+ return nullptr; // Satisfy compiler
+}
+
// Directory routines copied from src/win32/native/java/io/dirent_md.c
// * dirent_md.c 1.15 00/02/02
//
@@ -5829,57 +5835,6 @@ void* os::get_default_process_handle() {
return (void*)GetModuleHandle(nullptr);
}
-// Builds a platform dependent Agent_OnLoad_ function name
-// which is used to find statically linked in agents.
-// Parameters:
-// sym_name: Symbol in library we are looking for
-// lib_name: Name of library to look in, null for shared libs.
-// is_absolute_path == true if lib_name is absolute path to agent
-// such as "C:/a/b/L.dll"
-// == false if only the base name of the library is passed in
-// such as "L"
-char* os::build_agent_function_name(const char *sym_name, const char *lib_name,
- bool is_absolute_path) {
- char *agent_entry_name;
- size_t len;
- size_t name_len;
- size_t prefix_len = strlen(JNI_LIB_PREFIX);
- size_t suffix_len = strlen(JNI_LIB_SUFFIX);
- const char *start;
-
- if (lib_name != nullptr) {
- len = name_len = strlen(lib_name);
- if (is_absolute_path) {
- // Need to strip path, prefix and suffix
- if ((start = strrchr(lib_name, *os::file_separator())) != nullptr) {
- lib_name = ++start;
- } else {
- // Need to check for drive prefix
- if ((start = strchr(lib_name, ':')) != nullptr) {
- lib_name = ++start;
- }
- }
- if (len <= (prefix_len + suffix_len)) {
- return nullptr;
- }
- lib_name += prefix_len;
- name_len = strlen(lib_name) - suffix_len;
- }
- }
- len = (lib_name != nullptr ? name_len : 0) + strlen(sym_name) + 2;
- agent_entry_name = NEW_C_HEAP_ARRAY_RETURN_NULL(char, len, mtThread);
- if (agent_entry_name == nullptr) {
- return nullptr;
- }
-
- strcpy(agent_entry_name, sym_name);
- if (lib_name != nullptr) {
- strcat(agent_entry_name, "_");
- strncat(agent_entry_name, lib_name, name_len);
- }
- return agent_entry_name;
-}
-
/*
All the defined signal names for Windows.
diff --git a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
index 7702dbd17ad..1e3602d08f4 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
+++ b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
@@ -501,6 +501,7 @@ static inline void atomic_copy64(const volatile void *src, volatile void *dst) {
}
extern "C" {
+ // needs local assembler label '1:' to avoid trouble when using linktime optimization
int SpinPause() {
// We don't use StubRoutines::aarch64::spin_wait stub in order to
// avoid a costly call to os::current_thread_enable_wx() on MacOS.
@@ -523,14 +524,14 @@ extern "C" {
// to entry for case SpinWait::NOP
" add %[d], %[d], %[o] \n"
" br %[d] \n"
- " b SpinPause_return \n" // case SpinWait::NONE (-1)
+ " b 1f \n" // case SpinWait::NONE (-1)
" nop \n" // padding
" nop \n" // case SpinWait::NOP ( 0)
- " b SpinPause_return \n"
+ " b 1f \n"
" isb \n" // case SpinWait::ISB ( 1)
- " b SpinPause_return \n"
+ " b 1f \n"
" yield \n" // case SpinWait::YIELD ( 2)
- "SpinPause_return: \n"
+ "1: \n"
: [d]"=&r"(br_dst)
: [o]"r"(off)
: "memory");
diff --git a/src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp b/src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp
index 2020e2fdb24..f785d935393 100644
--- a/src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp
+++ b/src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp
@@ -181,6 +181,9 @@ void RiscvHwprobe::add_features_from_query_result() {
if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVFH)) {
VM_Version::ext_Zvfh.enable_feature();
}
+ if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZICOND)) {
+ VM_Version::ext_Zicond.enable_feature();
+ }
if (is_valid(RISCV_HWPROBE_KEY_CPUPERF_0)) {
VM_Version::unaligned_access.enable_feature(
query[RISCV_HWPROBE_KEY_CPUPERF_0].value & RISCV_HWPROBE_MISALIGNED_MASK);
diff --git a/src/hotspot/share/c1/c1_GraphBuilder.cpp b/src/hotspot/share/c1/c1_GraphBuilder.cpp
index 02be6f8d49e..0179923bc30 100644
--- a/src/hotspot/share/c1/c1_GraphBuilder.cpp
+++ b/src/hotspot/share/c1/c1_GraphBuilder.cpp
@@ -2121,7 +2121,7 @@ void GraphBuilder::invoke(Bytecodes::Code code) {
}
if (cha_monomorphic_target != nullptr) {
- assert(!target->can_be_statically_bound() || target->equals(cha_monomorphic_target), "");
+ assert(!target->can_be_statically_bound() || target == cha_monomorphic_target, "");
assert(!cha_monomorphic_target->is_abstract(), "");
if (!cha_monomorphic_target->can_be_statically_bound(actual_recv)) {
// If we inlined because CHA revealed only a single target method,
diff --git a/src/hotspot/share/cds/archiveUtils.cpp b/src/hotspot/share/cds/archiveUtils.cpp
index 4d717879e0f..3530fcff2b3 100644
--- a/src/hotspot/share/cds/archiveUtils.cpp
+++ b/src/hotspot/share/cds/archiveUtils.cpp
@@ -45,6 +45,7 @@
#include "utilities/debug.hpp"
#include "utilities/formatBuffer.hpp"
#include "utilities/globalDefinitions.hpp"
+#include "utilities/spinYield.hpp"
CHeapBitMap* ArchivePtrMarker::_ptrmap = nullptr;
CHeapBitMap* ArchivePtrMarker::_rw_ptrmap = nullptr;
@@ -399,3 +400,162 @@ size_t HeapRootSegments::segment_offset(size_t seg_idx) {
return _base_offset + seg_idx * _max_size_in_bytes;
}
+ArchiveWorkers::ArchiveWorkers() :
+ _end_semaphore(0),
+ _num_workers(max_workers()),
+ _started_workers(0),
+ _finish_tokens(0),
+ _state(UNUSED),
+ _task(nullptr) {}
+
+ArchiveWorkers::~ArchiveWorkers() {
+ assert(Atomic::load(&_state) != WORKING, "Should not be working");
+}
+
+int ArchiveWorkers::max_workers() {
+ // The pool is used for short-lived bursty tasks. We do not want to spend
+ // too much time creating and waking up threads unnecessarily. Plus, we do
+ // not want to overwhelm large machines. This is why we want to be very
+ // conservative about the number of workers actually needed.
+ return MAX2(0, log2i_graceful(os::active_processor_count()));
+}
+
+bool ArchiveWorkers::is_parallel() {
+ return _num_workers > 0;
+}
+
+void ArchiveWorkers::start_worker_if_needed() {
+ while (true) {
+ int cur = Atomic::load(&_started_workers);
+ if (cur >= _num_workers) {
+ return;
+ }
+ if (Atomic::cmpxchg(&_started_workers, cur, cur + 1, memory_order_relaxed) == cur) {
+ new ArchiveWorkerThread(this);
+ return;
+ }
+ }
+}
+
+void ArchiveWorkers::run_task(ArchiveWorkerTask* task) {
+ assert(Atomic::load(&_state) == UNUSED, "Should be unused yet");
+ assert(Atomic::load(&_task) == nullptr, "Should not have running tasks");
+ Atomic::store(&_state, WORKING);
+
+ if (is_parallel()) {
+ run_task_multi(task);
+ } else {
+ run_task_single(task);
+ }
+
+ assert(Atomic::load(&_state) == WORKING, "Should be working");
+ Atomic::store(&_state, SHUTDOWN);
+}
+
+void ArchiveWorkers::run_task_single(ArchiveWorkerTask* task) {
+ // Single thread needs no chunking.
+ task->configure_max_chunks(1);
+
+ // Execute the task ourselves, as there are no workers.
+ task->work(0, 1);
+}
+
+void ArchiveWorkers::run_task_multi(ArchiveWorkerTask* task) {
+ // Multiple threads can work with multiple chunks.
+ task->configure_max_chunks(_num_workers * CHUNKS_PER_WORKER);
+
+ // Set up the run and publish the task. Issue one additional finish token
+ // to cover the semaphore shutdown path, see below.
+ Atomic::store(&_finish_tokens, _num_workers + 1);
+ Atomic::release_store(&_task, task);
+
+ // Kick off pool startup by starting a single worker, and proceed
+ // immediately to executing the task locally.
+ start_worker_if_needed();
+
+ // Execute the task ourselves, while workers are catching up.
+ // This allows us to hide parts of task handoff latency.
+ task->run();
+
+ // Done executing task locally, wait for any remaining workers to complete.
+ // Once all workers report, we can proceed to termination. To do this safely,
+ // we need to make sure every worker has left. A spin-wait alone would suffice,
+ // but we do not want to burn cycles on it. A semaphore alone would not be safe,
+ // since workers can still be inside it as we proceed from wait here. So we block
+ // on semaphore first, and then spin-wait for all workers to terminate.
+ _end_semaphore.wait();
+ SpinYield spin;
+ while (Atomic::load(&_finish_tokens) != 0) {
+ spin.wait();
+ }
+
+ OrderAccess::fence();
+
+ assert(Atomic::load(&_finish_tokens) == 0, "All tokens are consumed");
+}
+
+void ArchiveWorkers::run_as_worker() {
+ assert(is_parallel(), "Should be in parallel mode");
+
+ ArchiveWorkerTask* task = Atomic::load_acquire(&_task);
+ task->run();
+
+ // All work done in threads should be visible to caller.
+ OrderAccess::fence();
+
+ // Signal the pool the work is complete, and we are exiting.
+ // Worker cannot do anything else with the pool after this.
+ if (Atomic::sub(&_finish_tokens, 1, memory_order_relaxed) == 1) {
+ // Last worker leaving. Notify the pool it can unblock to spin-wait.
+ // Then consume the last token and leave.
+ _end_semaphore.signal();
+ int last = Atomic::sub(&_finish_tokens, 1, memory_order_relaxed);
+ assert(last == 0, "Should be");
+ }
+}
+
+void ArchiveWorkerTask::run() {
+ while (true) {
+ int chunk = Atomic::load(&_chunk);
+ if (chunk >= _max_chunks) {
+ return;
+ }
+ if (Atomic::cmpxchg(&_chunk, chunk, chunk + 1, memory_order_relaxed) == chunk) {
+ assert(0 <= chunk && chunk < _max_chunks, "Sanity");
+ work(chunk, _max_chunks);
+ }
+ }
+}
+
+void ArchiveWorkerTask::configure_max_chunks(int max_chunks) {
+ if (_max_chunks == 0) {
+ _max_chunks = max_chunks;
+ }
+}
+
+ArchiveWorkerThread::ArchiveWorkerThread(ArchiveWorkers* pool) : NamedThread(), _pool(pool) {
+ set_name("ArchiveWorkerThread");
+ if (os::create_thread(this, os::os_thread)) {
+ os::start_thread(this);
+ } else {
+ vm_exit_during_initialization("Unable to create archive worker",
+ os::native_thread_creation_failed_msg());
+ }
+}
+
+void ArchiveWorkerThread::run() {
+ // Avalanche startup: each worker starts two others.
+ _pool->start_worker_if_needed();
+ _pool->start_worker_if_needed();
+
+ // Set ourselves up.
+ os::set_priority(this, NearMaxPriority);
+
+ // Work.
+ _pool->run_as_worker();
+}
+
+void ArchiveWorkerThread::post_run() {
+ this->NamedThread::post_run();
+ delete this;
+}
diff --git a/src/hotspot/share/cds/archiveUtils.hpp b/src/hotspot/share/cds/archiveUtils.hpp
index 723332c40e5..a10117e9f9a 100644
--- a/src/hotspot/share/cds/archiveUtils.hpp
+++ b/src/hotspot/share/cds/archiveUtils.hpp
@@ -33,6 +33,8 @@
#include "utilities/bitMap.hpp"
#include "utilities/exceptions.hpp"
#include "utilities/macros.hpp"
+#include "runtime/nonJavaThread.hpp"
+#include "runtime/semaphore.hpp"
class BootstrapInfo;
class ReservedSpace;
@@ -344,4 +346,74 @@ class HeapRootSegments {
HeapRootSegments& operator=(const HeapRootSegments&) = default;
};
+class ArchiveWorkers;
+
+// A task to be worked on by worker threads
+class ArchiveWorkerTask : public CHeapObj {
+ friend class ArchiveWorkers;
+private:
+ const char* _name;
+ int _max_chunks;
+ volatile int _chunk;
+
+ void run();
+
+ void configure_max_chunks(int max_chunks);
+
+public:
+ ArchiveWorkerTask(const char* name) :
+ _name(name), _max_chunks(0), _chunk(0) {}
+ const char* name() const { return _name; }
+ virtual void work(int chunk, int max_chunks) = 0;
+};
+
+class ArchiveWorkerThread : public NamedThread {
+ friend class ArchiveWorkers;
+private:
+ ArchiveWorkers* const _pool;
+
+ void post_run() override;
+
+public:
+ ArchiveWorkerThread(ArchiveWorkers* pool);
+ const char* type_name() const override { return "Archive Worker Thread"; }
+ void run() override;
+};
+
+// Special archive workers. The goal for this implementation is to startup fast,
+// distribute spiky workloads efficiently, and shutdown immediately after use.
+// This makes the implementation quite different from the normal GC worker pool.
+class ArchiveWorkers : public StackObj {
+ friend class ArchiveWorkerThread;
+private:
+ // Target number of chunks per worker. This should be large enough to even
+ // out work imbalance, and small enough to keep bookkeeping overheads low.
+ static constexpr int CHUNKS_PER_WORKER = 4;
+ static int max_workers();
+
+ Semaphore _end_semaphore;
+
+ int _num_workers;
+ int _started_workers;
+ int _finish_tokens;
+
+ typedef enum { UNUSED, WORKING, SHUTDOWN } State;
+ volatile State _state;
+
+ ArchiveWorkerTask* _task;
+
+ void run_as_worker();
+ void start_worker_if_needed();
+
+ void run_task_single(ArchiveWorkerTask* task);
+ void run_task_multi(ArchiveWorkerTask* task);
+
+ bool is_parallel();
+
+public:
+ ArchiveWorkers();
+ ~ArchiveWorkers();
+ void run_task(ArchiveWorkerTask* task);
+};
+
#endif // SHARE_CDS_ARCHIVEUTILS_HPP
diff --git a/src/hotspot/share/cds/cdsConfig.cpp b/src/hotspot/share/cds/cdsConfig.cpp
index ba5d12e6450..6ab77bf0007 100644
--- a/src/hotspot/share/cds/cdsConfig.cpp
+++ b/src/hotspot/share/cds/cdsConfig.cpp
@@ -250,9 +250,7 @@ void CDSConfig::init_shared_archive_paths() {
}
void CDSConfig::check_internal_module_property(const char* key, const char* value) {
- if (Arguments::is_internal_module_property(key) &&
- !Arguments::is_module_path_property(key) &&
- !Arguments::is_add_modules_property(key)) {
+ if (Arguments::is_incompatible_cds_internal_module_property(key)) {
stop_using_optimized_module_handling();
log_info(cds)("optimized module handling: disabled due to incompatible property: %s=%s", key, value);
}
diff --git a/src/hotspot/share/cds/cds_globals.hpp b/src/hotspot/share/cds/cds_globals.hpp
index 38f5d8f46a6..811740cfbcb 100644
--- a/src/hotspot/share/cds/cds_globals.hpp
+++ b/src/hotspot/share/cds/cds_globals.hpp
@@ -117,7 +117,10 @@
product(bool, AOTClassLinking, false, \
"Load/link all archived classes for the boot/platform/app " \
"loaders before application main") \
-
+ \
+ product(bool, AOTCacheParallelRelocation, true, DIAGNOSTIC, \
+ "Use parallel relocation code to speed up startup.") \
+ \
// end of CDS_FLAGS
DECLARE_FLAGS(CDS_FLAGS)
diff --git a/src/hotspot/share/cds/filemap.cpp b/src/hotspot/share/cds/filemap.cpp
index 91a2a57dee5..594d8817322 100644
--- a/src/hotspot/share/cds/filemap.cpp
+++ b/src/hotspot/share/cds/filemap.cpp
@@ -1975,6 +1975,32 @@ char* FileMapInfo::map_bitmap_region() {
return bitmap_base;
}
+class SharedDataRelocationTask : public ArchiveWorkerTask {
+private:
+ BitMapView* const _rw_bm;
+ BitMapView* const _ro_bm;
+ SharedDataRelocator* const _rw_reloc;
+ SharedDataRelocator* const _ro_reloc;
+
+public:
+ SharedDataRelocationTask(BitMapView* rw_bm, BitMapView* ro_bm, SharedDataRelocator* rw_reloc, SharedDataRelocator* ro_reloc) :
+ ArchiveWorkerTask("Shared Data Relocation"),
+ _rw_bm(rw_bm), _ro_bm(ro_bm), _rw_reloc(rw_reloc), _ro_reloc(ro_reloc) {}
+
+ void work(int chunk, int max_chunks) override {
+ work_on(chunk, max_chunks, _rw_bm, _rw_reloc);
+ work_on(chunk, max_chunks, _ro_bm, _ro_reloc);
+ }
+
+ void work_on(int chunk, int max_chunks, BitMapView* bm, SharedDataRelocator* reloc) {
+ BitMap::idx_t size = bm->size();
+ BitMap::idx_t start = MIN2(size, size * chunk / max_chunks);
+ BitMap::idx_t end = MIN2(size, size * (chunk + 1) / max_chunks);
+ assert(end > start, "Sanity: no empty slices");
+ bm->iterate(reloc, start, end);
+ }
+};
+
// This is called when we cannot map the archive at the requested[ base address (usually 0x800000000).
// We relocate all pointers in the 2 core regions (ro, rw).
bool FileMapInfo::relocate_pointers_in_core_regions(intx addr_delta) {
@@ -2013,8 +2039,15 @@ bool FileMapInfo::relocate_pointers_in_core_regions(intx addr_delta) {
valid_new_base, valid_new_end, addr_delta);
SharedDataRelocator ro_patcher((address*)ro_patch_base + header()->ro_ptrmap_start_pos(), (address*)ro_patch_end, valid_old_base, valid_old_end,
valid_new_base, valid_new_end, addr_delta);
- rw_ptrmap.iterate(&rw_patcher);
- ro_ptrmap.iterate(&ro_patcher);
+
+ if (AOTCacheParallelRelocation) {
+ ArchiveWorkers workers;
+ SharedDataRelocationTask task(&rw_ptrmap, &ro_ptrmap, &rw_patcher, &ro_patcher);
+ workers.run_task(&task);
+ } else {
+ rw_ptrmap.iterate(&rw_patcher);
+ ro_ptrmap.iterate(&ro_patcher);
+ }
// The MetaspaceShared::bm region will be unmapped in MetaspaceShared::initialize_shared_spaces().
@@ -2698,8 +2731,8 @@ ClassFileStream* FileMapInfo::get_stream_from_class_loader(Handle class_loader,
const char* file_name,
TRAPS) {
JavaValue result(T_OBJECT);
- TempNewSymbol class_name_sym = SymbolTable::new_symbol(file_name);
- Handle ext_class_name = java_lang_String::externalize_classname(class_name_sym, CHECK_NULL);
+ oop class_name = java_lang_String::create_oop_from_str(file_name, THREAD);
+ Handle h_class_name = Handle(THREAD, class_name);
// byte[] ClassLoader.getResourceAsByteArray(String name)
JavaCalls::call_virtual(&result,
@@ -2707,7 +2740,7 @@ ClassFileStream* FileMapInfo::get_stream_from_class_loader(Handle class_loader,
vmClasses::ClassLoader_klass(),
vmSymbols::getResourceAsByteArray_name(),
vmSymbols::getResourceAsByteArray_signature(),
- ext_class_name,
+ h_class_name,
CHECK_NULL);
assert(result.get_type() == T_OBJECT, "just checking");
oop obj = result.get_oop();
diff --git a/src/hotspot/share/cds/metaspaceShared.cpp b/src/hotspot/share/cds/metaspaceShared.cpp
index 3ea2f54cfa9..f21b9c9060d 100644
--- a/src/hotspot/share/cds/metaspaceShared.cpp
+++ b/src/hotspot/share/cds/metaspaceShared.cpp
@@ -281,7 +281,7 @@ void MetaspaceShared::initialize_for_static_dump() {
SharedBaseAddress = (size_t)_requested_base_address;
size_t symbol_rs_size = LP64_ONLY(3 * G) NOT_LP64(128 * M);
- _symbol_rs = ReservedSpace(symbol_rs_size);
+ _symbol_rs = ReservedSpace(symbol_rs_size, mtClassShared);
if (!_symbol_rs.is_reserved()) {
log_error(cds)("Unable to reserve memory for symbols: " SIZE_FORMAT " bytes.", symbol_rs_size);
MetaspaceShared::unrecoverable_writing_error();
@@ -423,8 +423,7 @@ void MetaspaceShared::write_method_handle_intrinsics() {
void MetaspaceShared::early_serialize(SerializeClosure* soc) {
int tag = 0;
soc->do_tag(--tag);
- CDS_JAVA_HEAP_ONLY(Modules::serialize(soc);)
- CDS_JAVA_HEAP_ONLY(Modules::serialize_addmods_names(soc);)
+ CDS_JAVA_HEAP_ONLY(Modules::serialize_archived_module_info(soc);)
soc->do_tag(666);
}
@@ -567,10 +566,7 @@ class StaticArchiveBuilder : public ArchiveBuilder {
char* VM_PopulateDumpSharedSpace::dump_early_read_only_tables() {
ArchiveBuilder::OtherROAllocMark mark;
- // Write module name into archive
- CDS_JAVA_HEAP_ONLY(Modules::dump_main_module_name();)
- // Write module names from --add-modules into archive
- CDS_JAVA_HEAP_ONLY(Modules::dump_addmods_names();)
+ CDS_JAVA_HEAP_ONLY(Modules::dump_archived_module_info());
DumpRegion* ro_region = ArchiveBuilder::current()->ro_region();
char* start = ro_region->top();
@@ -1014,9 +1010,7 @@ void VM_PopulateDumpSharedSpace::dump_java_heap_objects(GrowableArray* k
Klass* k = klasses->at(i);
if (k->is_instance_klass()) {
InstanceKlass* ik = InstanceKlass::cast(k);
- if (ik->is_linked()) {
- ik->constants()->add_dumped_interned_strings();
- }
+ ik->constants()->add_dumped_interned_strings();
}
}
if (_extra_interned_strings != nullptr) {
diff --git a/src/hotspot/share/ci/ciMethod.cpp b/src/hotspot/share/ci/ciMethod.cpp
index a74a812c6a2..80277b91d22 100644
--- a/src/hotspot/share/ci/ciMethod.cpp
+++ b/src/hotspot/share/ci/ciMethod.cpp
@@ -742,6 +742,13 @@ ciMethod* ciMethod::find_monomorphic_target(ciInstanceKlass* caller,
if (target() == nullptr) {
return nullptr;
}
+
+ // Redefinition support.
+ if (this->is_old() || root_m->is_old() || target->is_old()) {
+ guarantee(CURRENT_THREAD_ENV->jvmti_state_changed(), "old method not detected");
+ return nullptr;
+ }
+
if (target() == root_m->get_Method()) {
return root_m;
}
@@ -781,22 +788,6 @@ bool ciMethod::can_omit_stack_trace() const {
return _can_omit_stack_trace;
}
-// ------------------------------------------------------------------
-// ciMethod::equals
-//
-// Returns true if the methods are the same, taking redefined methods
-// into account.
-bool ciMethod::equals(const ciMethod* m) const {
- if (this == m) return true;
- VM_ENTRY_MARK;
- Method* m1 = this->get_Method();
- Method* m2 = m->get_Method();
- if (m1->is_old()) m1 = m1->get_new_method();
- if (m2->is_old()) m2 = m2->get_new_method();
- return m1 == m2;
-}
-
-
// ------------------------------------------------------------------
// ciMethod::resolve_invoke
//
@@ -835,6 +826,12 @@ ciMethod* ciMethod::resolve_invoke(ciKlass* caller, ciKlass* exact_receiver, boo
ciMethod* result = this;
if (m != get_Method()) {
+ // Redefinition support.
+ if (this->is_old() || m->is_old()) {
+ guarantee(CURRENT_THREAD_ENV->jvmti_state_changed(), "old method not detected");
+ return nullptr;
+ }
+
result = CURRENT_THREAD_ENV->get_method(m);
}
@@ -1495,3 +1492,10 @@ bool ciMethod::is_consistent_info(ciMethod* declared_method, ciMethod* resolved_
}
// ------------------------------------------------------------------
+// ciMethod::is_old
+//
+// Return true for redefined methods
+bool ciMethod::is_old() const {
+ ASSERT_IN_VM;
+ return get_Method()->is_old();
+}
diff --git a/src/hotspot/share/ci/ciMethod.hpp b/src/hotspot/share/ci/ciMethod.hpp
index cc524930192..eecd9427585 100644
--- a/src/hotspot/share/ci/ciMethod.hpp
+++ b/src/hotspot/share/ci/ciMethod.hpp
@@ -360,13 +360,12 @@ class ciMethod : public ciMetadata {
bool is_vector_method() const;
bool is_object_initializer() const;
bool is_scoped() const;
+ bool is_old() const;
bool can_be_statically_bound(ciInstanceKlass* context) const;
bool can_omit_stack_trace() const;
- bool equals(const ciMethod* m) const;
-
// Replay data methods
static void dump_name_as_ascii(outputStream* st, Method* method);
void dump_name_as_ascii(outputStream* st);
diff --git a/src/hotspot/share/classfile/classLoader.cpp b/src/hotspot/share/classfile/classLoader.cpp
index 1b34f2ebede..aac312c36a3 100644
--- a/src/hotspot/share/classfile/classLoader.cpp
+++ b/src/hotspot/share/classfile/classLoader.cpp
@@ -949,16 +949,35 @@ void* ClassLoader::dll_lookup(void* lib, const char* name, const char* path) {
void ClassLoader::load_java_library() {
assert(CanonicalizeEntry == nullptr, "should not load java library twice");
+ if (is_vm_statically_linked()) {
+ CanonicalizeEntry = CAST_TO_FN_PTR(canonicalize_fn_t, os::lookup_function("JDK_Canonicalize"));
+ assert(CanonicalizeEntry != nullptr, "could not lookup JDK_Canonicalize");
+ return;
+ }
+
void *javalib_handle = os::native_java_library();
if (javalib_handle == nullptr) {
vm_exit_during_initialization("Unable to load java library", nullptr);
}
CanonicalizeEntry = CAST_TO_FN_PTR(canonicalize_fn_t, dll_lookup(javalib_handle, "JDK_Canonicalize", nullptr));
+ assert(CanonicalizeEntry != nullptr, "could not lookup JDK_Canonicalize in java library");
}
void ClassLoader::load_jimage_library() {
assert(JImageOpen == nullptr, "should not load jimage library twice");
+
+ if (is_vm_statically_linked()) {
+ JImageOpen = CAST_TO_FN_PTR(JImageOpen_t, os::lookup_function("JIMAGE_Open"));
+ JImageClose = CAST_TO_FN_PTR(JImageClose_t, os::lookup_function("JIMAGE_Close"));
+ JImageFindResource = CAST_TO_FN_PTR(JImageFindResource_t, os::lookup_function("JIMAGE_FindResource"));
+ JImageGetResource = CAST_TO_FN_PTR(JImageGetResource_t, os::lookup_function("JIMAGE_GetResource"));
+ assert(JImageOpen != nullptr && JImageClose != nullptr &&
+ JImageFindResource != nullptr && JImageGetResource != nullptr,
+ "could not lookup all jimage library functions");
+ return;
+ }
+
char path[JVM_MAXPATHLEN];
char ebuf[1024];
void* handle = nullptr;
@@ -973,6 +992,9 @@ void ClassLoader::load_jimage_library() {
JImageClose = CAST_TO_FN_PTR(JImageClose_t, dll_lookup(handle, "JIMAGE_Close", path));
JImageFindResource = CAST_TO_FN_PTR(JImageFindResource_t, dll_lookup(handle, "JIMAGE_FindResource", path));
JImageGetResource = CAST_TO_FN_PTR(JImageGetResource_t, dll_lookup(handle, "JIMAGE_GetResource", path));
+ assert(JImageOpen != nullptr && JImageClose != nullptr &&
+ JImageFindResource != nullptr && JImageGetResource != nullptr,
+ "could not lookup all jimage library functions in jimage library");
}
int ClassLoader::crc32(int crc, const char* buf, int len) {
diff --git a/src/hotspot/share/classfile/classLoaderHierarchyDCmd.hpp b/src/hotspot/share/classfile/classLoaderHierarchyDCmd.hpp
index 26c765443c9..cd89c9cb978 100644
--- a/src/hotspot/share/classfile/classLoaderHierarchyDCmd.hpp
+++ b/src/hotspot/share/classfile/classLoaderHierarchyDCmd.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -46,11 +46,6 @@ class ClassLoaderHierarchyDCmd: public DCmdWithParser {
static const char* impact() {
return "Medium: Depends on number of class loaders and classes loaded.";
}
- static const JavaPermission permission() {
- JavaPermission p = {"java.lang.management.ManagementPermission",
- "monitor", nullptr};
- return p;
- }
static int num_arguments() { return 3; }
virtual void execute(DCmdSource source, TRAPS);
diff --git a/src/hotspot/share/classfile/classLoaderStats.hpp b/src/hotspot/share/classfile/classLoaderStats.hpp
index 4818ddff609..06d375b3e9b 100644
--- a/src/hotspot/share/classfile/classLoaderStats.hpp
+++ b/src/hotspot/share/classfile/classLoaderStats.hpp
@@ -58,12 +58,6 @@ class ClassLoaderStatsDCmd : public DCmd {
static int num_arguments() {
return 0;
}
-
- static const JavaPermission permission() {
- JavaPermission p = {"java.lang.management.ManagementPermission",
- "monitor", nullptr};
- return p;
- }
};
diff --git a/src/hotspot/share/classfile/modules.cpp b/src/hotspot/share/classfile/modules.cpp
index 2a8f8bd2424..9c430e77d09 100644
--- a/src/hotspot/share/classfile/modules.cpp
+++ b/src/hotspot/share/classfile/modules.cpp
@@ -562,13 +562,55 @@ void Modules::verify_archived_modules() {
char* Modules::_archived_main_module_name = nullptr;
char* Modules::_archived_addmods_names = nullptr;
+char* Modules::_archived_native_access_flags = nullptr;
void Modules::dump_main_module_name() {
const char* module_name = Arguments::get_property("jdk.module.main");
if (module_name != nullptr) {
_archived_main_module_name = ArchiveBuilder::current()->ro_strdup(module_name);
}
- ArchivePtrMarker::mark_pointer(&_archived_main_module_name);
+}
+
+void Modules::check_archived_flag_consistency(char* archived_flag, const char* runtime_flag, const char* property) {
+ log_info(cds)("%s %s", property,
+ archived_flag != nullptr ? archived_flag : "(null)");
+ bool disable = false;
+ if (runtime_flag == nullptr) {
+ if (archived_flag != nullptr) {
+ log_info(cds)("Mismatched values for property %s: %s specified during dump time but not during runtime", property, archived_flag);
+ disable = true;
+ }
+ } else {
+ if (archived_flag == nullptr) {
+ log_info(cds)("Mismatched values for property %s: %s specified during runtime but not during dump time", property, runtime_flag);
+ disable = true;
+ } else if (strcmp(runtime_flag, archived_flag) != 0) {
+ log_info(cds)("Mismatched values for property %s: runtime %s dump time %s", property, runtime_flag, archived_flag);
+ disable = true;
+ }
+ }
+
+ if (disable) {
+ log_info(cds)("Disabling optimized module handling");
+ CDSConfig::stop_using_optimized_module_handling();
+ }
+ log_info(cds)("optimized module handling: %s", CDSConfig::is_using_optimized_module_handling() ? "enabled" : "disabled");
+ log_info(cds)("full module graph: %s", CDSConfig::is_using_full_module_graph() ? "enabled" : "disabled");
+}
+
+void Modules::dump_archived_module_info() {
+ // Write module name into archive
+ CDS_JAVA_HEAP_ONLY(Modules::dump_main_module_name();)
+ // Write module names from --add-modules into archive
+ CDS_JAVA_HEAP_ONLY(Modules::dump_addmods_names();)
+ // Write native enable-native-access flag into archive
+ CDS_JAVA_HEAP_ONLY(Modules::dump_native_access_flag());
+}
+
+void Modules::serialize_archived_module_info(SerializeClosure* soc) {
+ CDS_JAVA_HEAP_ONLY(Modules::serialize(soc);)
+ CDS_JAVA_HEAP_ONLY(Modules::serialize_addmods_names(soc);)
+ CDS_JAVA_HEAP_ONLY(Modules::serialize_native_access_flags(soc);)
}
void Modules::serialize(SerializeClosure* soc) {
@@ -577,89 +619,71 @@ void Modules::serialize(SerializeClosure* soc) {
const char* runtime_main_module = Arguments::get_property("jdk.module.main");
log_info(cds)("_archived_main_module_name %s",
_archived_main_module_name != nullptr ? _archived_main_module_name : "(null)");
- bool disable = false;
- if (runtime_main_module == nullptr) {
- if (_archived_main_module_name != nullptr) {
- log_info(cds)("Module %s specified during dump time but not during runtime", _archived_main_module_name);
- disable = true;
- }
- } else {
- if (_archived_main_module_name == nullptr) {
- log_info(cds)("Module %s specified during runtime but not during dump time", runtime_main_module);
- disable = true;
- } else if (strcmp(runtime_main_module, _archived_main_module_name) != 0) {
- log_info(cds)("Mismatched modules: runtime %s dump time %s", runtime_main_module, _archived_main_module_name);
- disable = true;
- }
- }
- if (disable) {
- log_info(cds)("Disabling optimized module handling");
- CDSConfig::stop_using_optimized_module_handling();
- }
- log_info(cds)("optimized module handling: %s", CDSConfig::is_using_optimized_module_handling() ? "enabled" : "disabled");
- log_info(cds)("full module graph: %s", CDSConfig::is_using_full_module_graph() ? "enabled" : "disabled");
+ check_archived_flag_consistency(_archived_main_module_name, runtime_main_module, "jdk.module.main");
// Don't hold onto the pointer, in case we might decide to unmap the archive.
_archived_main_module_name = nullptr;
}
}
+void Modules::dump_native_access_flag() {
+ const char* native_access_names = get_native_access_flags_as_sorted_string();
+ if (native_access_names != nullptr) {
+ _archived_native_access_flags = ArchiveBuilder::current()->ro_strdup(native_access_names);
+ }
+}
+
+const char* Modules::get_native_access_flags_as_sorted_string() {
+ return get_numbered_property_as_sorted_string("jdk.module.enable.native.access");
+}
+
+void Modules::serialize_native_access_flags(SerializeClosure* soc) {
+ soc->do_ptr(&_archived_native_access_flags);
+ if (soc->reading()) {
+ check_archived_flag_consistency(_archived_native_access_flags, get_native_access_flags_as_sorted_string(), "jdk.module.enable.native.access");
+
+ // Don't hold onto the pointer, in case we might decide to unmap the archive.
+ _archived_native_access_flags = nullptr;
+ }
+}
+
void Modules::dump_addmods_names() {
- unsigned int count = Arguments::addmods_count();
const char* addmods_names = get_addmods_names_as_sorted_string();
if (addmods_names != nullptr) {
_archived_addmods_names = ArchiveBuilder::current()->ro_strdup(addmods_names);
}
- ArchivePtrMarker::mark_pointer(&_archived_addmods_names);
+}
+
+const char* Modules::get_addmods_names_as_sorted_string() {
+ return get_numbered_property_as_sorted_string("jdk.module.addmods");
}
void Modules::serialize_addmods_names(SerializeClosure* soc) {
soc->do_ptr(&_archived_addmods_names);
if (soc->reading()) {
- bool disable = false;
- if (_archived_addmods_names[0] != '\0') {
- if (Arguments::addmods_count() == 0) {
- log_info(cds)("--add-modules module name(s) found in archive but not specified during runtime: %s",
- _archived_addmods_names);
- disable = true;
- } else {
- const char* addmods_names = get_addmods_names_as_sorted_string();
- if (strcmp((const char*)_archived_addmods_names, addmods_names) != 0) {
- log_info(cds)("Mismatched --add-modules module name(s).");
- log_info(cds)(" dump time: %s runtime: %s", _archived_addmods_names, addmods_names);
- disable = true;
- }
- }
- } else {
- if (Arguments::addmods_count() > 0) {
- log_info(cds)("--add-modules module name(s) specified during runtime but not found in archive: %s",
- get_addmods_names_as_sorted_string());
- disable = true;
- }
- }
- if (disable) {
- log_info(cds)("Disabling optimized module handling");
- CDSConfig::stop_using_optimized_module_handling();
- }
- log_info(cds)("optimized module handling: %s", CDSConfig::is_using_optimized_module_handling() ? "enabled" : "disabled");
- log_info(cds)("full module graph: %s", CDSConfig::is_using_full_module_graph() ? "enabled" : "disabled");
+ check_archived_flag_consistency(_archived_addmods_names, get_addmods_names_as_sorted_string(), "jdk.module.addmods");
// Don't hold onto the pointer, in case we might decide to unmap the archive.
_archived_addmods_names = nullptr;
}
}
-const char* Modules::get_addmods_names_as_sorted_string() {
+const char* Modules::get_numbered_property_as_sorted_string(const char* property) {
ResourceMark rm;
- const int max_digits = 3;
+ // theoretical string size limit for decimal int, but the following loop will end much sooner due to
+ // OS command-line size limit.
+ const int max_digits = 10;
const int extra_symbols_count = 2; // includes '.', '\0'
- size_t prop_len = strlen("jdk.module.addmods") + max_digits + extra_symbols_count;
+ size_t prop_len = strlen(property) + max_digits + extra_symbols_count;
char* prop_name = resource_allocate_bytes(prop_len);
GrowableArray list;
- for (unsigned int i = 0; i < Arguments::addmods_count(); i++) {
- jio_snprintf(prop_name, prop_len, "jdk.module.addmods.%d", i);
+ for (unsigned int i = 0;; i++) {
+ jio_snprintf(prop_name, prop_len, "%s.%d", property, i);
const char* prop_value = Arguments::get_property(prop_name);
+ if (prop_value == nullptr) {
+ break;
+ }
char* p = resource_allocate_bytes(strlen(prop_value) + 1);
strcpy(p, prop_value);
while (*p == ',') p++; // skip leading commas
@@ -695,11 +719,12 @@ const char* Modules::get_addmods_names_as_sorted_string() {
if (strcmp(m, last_string) != 0) { // filter out duplicates
st.print("%s%s", prefix, m);
last_string = m;
- prefix = "\n";
+ prefix = ",";
}
}
- return (const char*)os::strdup(st.as_string()); // Example: "java.base,java.compiler"
+ const char* result = (const char*)os::strdup(st.as_string()); // Example: "java.base,java.compiler"
+ return strcmp(result, "") != 0 ? result : nullptr;
}
void Modules::define_archived_modules(Handle h_platform_loader, Handle h_system_loader, TRAPS) {
diff --git a/src/hotspot/share/classfile/modules.hpp b/src/hotspot/share/classfile/modules.hpp
index 3ef6f57a57b..03069bd3452 100644
--- a/src/hotspot/share/classfile/modules.hpp
+++ b/src/hotspot/share/classfile/modules.hpp
@@ -1,5 +1,5 @@
/*
-* Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+* Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -59,15 +59,25 @@ class Modules : AllStatic {
static void define_archived_modules(Handle h_platform_loader, Handle h_system_loader,
TRAPS) NOT_CDS_JAVA_HEAP_RETURN;
static void verify_archived_modules() NOT_CDS_JAVA_HEAP_RETURN;
+ static void dump_archived_module_info() NOT_CDS_JAVA_HEAP_RETURN;
+ static void serialize_archived_module_info(SerializeClosure* soc) NOT_CDS_JAVA_HEAP_RETURN;
static void dump_main_module_name() NOT_CDS_JAVA_HEAP_RETURN;
static void serialize(SerializeClosure* soc) NOT_CDS_JAVA_HEAP_RETURN;
+ static void check_archived_flag_consistency(char* archived_flag, const char* runtime_flag, const char* property) NOT_CDS_JAVA_HEAP_RETURN;
+
+ static void dump_native_access_flag() NOT_CDS_JAVA_HEAP_RETURN;
+ static const char* get_native_access_flags_as_sorted_string() NOT_CDS_JAVA_HEAP_RETURN_(nullptr);
+ static void serialize_native_access_flags(SerializeClosure* soc) NOT_CDS_JAVA_HEAP_RETURN;
+
static void dump_addmods_names() NOT_CDS_JAVA_HEAP_RETURN;
- static void serialize_addmods_names(SerializeClosure* soc) NOT_CDS_JAVA_HEAP_RETURN;
static const char* get_addmods_names_as_sorted_string() NOT_CDS_JAVA_HEAP_RETURN_(nullptr);
+ static void serialize_addmods_names(SerializeClosure* soc) NOT_CDS_JAVA_HEAP_RETURN;
+ static const char* get_numbered_property_as_sorted_string(const char* property) NOT_CDS_JAVA_HEAP_RETURN_(nullptr);
#if INCLUDE_CDS_JAVA_HEAP
static char* _archived_main_module_name;
static char* _archived_addmods_names;
+ static char* _archived_native_access_flags;
#endif
// Provides the java.lang.Module for the unnamed module defined
diff --git a/src/hotspot/share/classfile/verifier.cpp b/src/hotspot/share/classfile/verifier.cpp
index 9a36584916d..0ac2cc350b1 100644
--- a/src/hotspot/share/classfile/verifier.cpp
+++ b/src/hotspot/share/classfile/verifier.cpp
@@ -85,15 +85,20 @@ static verify_byte_codes_fn_t verify_byte_codes_fn() {
if (_verify_byte_codes_fn != nullptr)
return _verify_byte_codes_fn;
+ void *lib_handle = nullptr;
// Load verify dll
- char buffer[JVM_MAXPATHLEN];
- char ebuf[1024];
- if (!os::dll_locate_lib(buffer, sizeof(buffer), Arguments::get_dll_dir(), "verify"))
- return nullptr; // Caller will throw VerifyError
+ if (is_vm_statically_linked()) {
+ lib_handle = os::get_default_process_handle();
+ } else {
+ char buffer[JVM_MAXPATHLEN];
+ char ebuf[1024];
+ if (!os::dll_locate_lib(buffer, sizeof(buffer), Arguments::get_dll_dir(), "verify"))
+ return nullptr; // Caller will throw VerifyError
- void *lib_handle = os::dll_load(buffer, ebuf, sizeof(ebuf));
- if (lib_handle == nullptr)
- return nullptr; // Caller will throw VerifyError
+ lib_handle = os::dll_load(buffer, ebuf, sizeof(ebuf));
+ if (lib_handle == nullptr)
+ return nullptr; // Caller will throw VerifyError
+ }
void *fn = os::dll_lookup(lib_handle, "VerifyClassForMajorVersion");
if (fn == nullptr)
diff --git a/src/hotspot/share/classfile/vmSymbols.hpp b/src/hotspot/share/classfile/vmSymbols.hpp
index 6a6f7754c50..46c156a5445 100644
--- a/src/hotspot/share/classfile/vmSymbols.hpp
+++ b/src/hotspot/share/classfile/vmSymbols.hpp
@@ -739,10 +739,15 @@ class SerializeClosure;
template(toFileURL_signature, "(Ljava/lang/String;)Ljava/net/URL;") \
template(url_array_classloader_void_signature, "([Ljava/net/URL;Ljava/lang/ClassLoader;)V") \
\
- /* Thread.dump_to_file jcmd */ \
+ /* jcmd Thread.dump_to_file */ \
template(jdk_internal_vm_ThreadDumper, "jdk/internal/vm/ThreadDumper") \
template(dumpThreads_name, "dumpThreads") \
template(dumpThreadsToJson_name, "dumpThreadsToJson") \
+ \
+ /* jcmd Thread.vthread_scheduler and Thread.vthread_pollers */ \
+ template(jdk_internal_vm_JcmdVThreadCommands, "jdk/internal/vm/JcmdVThreadCommands") \
+ template(printScheduler_name, "printScheduler") \
+ template(printPollers_name, "printPollers") \
/*end*/
diff --git a/src/hotspot/share/compiler/compileBroker.cpp b/src/hotspot/share/compiler/compileBroker.cpp
index a7b0627c0a4..30956b6c793 100644
--- a/src/hotspot/share/compiler/compileBroker.cpp
+++ b/src/hotspot/share/compiler/compileBroker.cpp
@@ -1448,7 +1448,7 @@ nmethod* CompileBroker::compile_method(const methodHandle& method, int osr_bci,
// do the compilation
if (method->is_native()) {
if (!PreferInterpreterNativeStubs || method->is_method_handle_intrinsic()) {
-#if defined(X86) && !defined(ZERO)
+#if defined(IA32) && !defined(ZERO)
// The following native methods:
//
// java.lang.Float.intBitsToFloat
@@ -1470,7 +1470,7 @@ nmethod* CompileBroker::compile_method(const methodHandle& method, int osr_bci,
method->intrinsic_id() == vmIntrinsics::_doubleToRawLongBits))) {
return nullptr;
}
-#endif // X86 && !ZERO
+#endif // IA32 && !ZERO
// To properly handle the appendix argument for out-of-line calls we are using a small trampoline that
// pops off the appendix argument and jumps to the target (see gen_special_dispatch in SharedRuntime).
diff --git a/src/hotspot/share/compiler/methodMatcher.cpp b/src/hotspot/share/compiler/methodMatcher.cpp
index 1a0ade2fadb..0bd5cdd8501 100644
--- a/src/hotspot/share/compiler/methodMatcher.cpp
+++ b/src/hotspot/share/compiler/methodMatcher.cpp
@@ -219,21 +219,22 @@ bool MethodMatcher::match(Symbol* candidate, Symbol* match, Mode match_mode) con
static MethodMatcher::Mode check_mode(char name[], const char*& error_msg) {
int match = MethodMatcher::Exact;
+ size_t len = strlen(name);
if (name[0] == '*') {
- if (strlen(name) == 1) {
+ if (len == 1) {
return MethodMatcher::Any;
}
match |= MethodMatcher::Suffix;
- memmove(name, name + 1, strlen(name + 1) + 1);
+ memmove(name, name + 1, len); // Include terminating nul in move.
+ len--;
}
- size_t len = strlen(name);
if (len > 0 && name[len - 1] == '*') {
match |= MethodMatcher::Prefix;
name[--len] = '\0';
}
- if (strlen(name) == 0) {
+ if (len == 0) {
error_msg = "** Not a valid pattern";
return MethodMatcher::Any;
}
diff --git a/src/hotspot/share/gc/g1/g1HeapRegionRemSet.cpp b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.cpp
index fe1590b94a8..c1343fd7dbf 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegionRemSet.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.cpp
@@ -22,8 +22,6 @@
*
*/
-#include
-
#include "precompiled.hpp"
#include "gc/g1/g1BlockOffsetTable.inline.hpp"
#include "gc/g1/g1CardSetContainers.inline.hpp"
diff --git a/src/hotspot/share/gc/parallel/objectStartArray.cpp b/src/hotspot/share/gc/parallel/objectStartArray.cpp
index ef9de7abfd7..2a0f12ec70e 100644
--- a/src/hotspot/share/gc/parallel/objectStartArray.cpp
+++ b/src/hotspot/share/gc/parallel/objectStartArray.cpp
@@ -47,11 +47,10 @@ void ObjectStartArray::initialize(MemRegion reserved_region) {
// Do not use large-pages for the backing store. The one large page region
// will be used for the heap proper.
- ReservedSpace backing_store(bytes_to_reserve);
+ ReservedSpace backing_store(bytes_to_reserve, mtGC);
if (!backing_store.is_reserved()) {
vm_exit_during_initialization("Could not reserve space for ObjectStartArray");
}
- MemTracker::record_virtual_memory_tag(backing_store.base(), mtGC);
// We do not commit any memory initially
_virtual_space.initialize(backing_store);
diff --git a/src/hotspot/share/gc/serial/serialBlockOffsetTable.cpp b/src/hotspot/share/gc/serial/serialBlockOffsetTable.cpp
index 31f18652c63..7ac0fcc8b53 100644
--- a/src/hotspot/share/gc/serial/serialBlockOffsetTable.cpp
+++ b/src/hotspot/share/gc/serial/serialBlockOffsetTable.cpp
@@ -37,13 +37,11 @@ SerialBlockOffsetTable::SerialBlockOffsetTable(MemRegion reserved,
size_t init_word_size):
_reserved(reserved) {
size_t size = compute_size(reserved.word_size());
- ReservedSpace rs(size);
+ ReservedSpace rs(size, mtGC);
if (!rs.is_reserved()) {
vm_exit_during_initialization("Could not reserve enough space for heap offset array");
}
- MemTracker::record_virtual_memory_tag((address)rs.base(), mtGC);
-
if (!_vs.initialize(rs, 0)) {
vm_exit_during_initialization("Could not reserve enough space for heap offset array");
}
diff --git a/src/hotspot/share/gc/serial/tenuredGeneration.cpp b/src/hotspot/share/gc/serial/tenuredGeneration.cpp
index 99031c379d8..a00eb369980 100644
--- a/src/hotspot/share/gc/serial/tenuredGeneration.cpp
+++ b/src/hotspot/share/gc/serial/tenuredGeneration.cpp
@@ -378,11 +378,14 @@ void TenuredGeneration::update_counters() {
bool TenuredGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const {
size_t available = _the_space->free() + _virtual_space.uncommitted_size();
- size_t av_promo = (size_t)_avg_promoted->padded_average();
- bool res = (available >= av_promo) || (available >= max_promotion_in_bytes);
+
+ size_t avg_promoted = (size_t)_avg_promoted->padded_average();
+ size_t promotion_estimate = MIN2(avg_promoted, max_promotion_in_bytes);
+
+ bool res = (promotion_estimate <= available);
log_trace(gc)("Tenured: promo attempt is%s safe: available(" SIZE_FORMAT ") %s av_promo(" SIZE_FORMAT "), max_promo(" SIZE_FORMAT ")",
- res? "":" not", available, res? ">=":"<", av_promo, max_promotion_in_bytes);
+ res? "":" not", available, res? ">=":"<", avg_promoted, max_promotion_in_bytes);
return res;
}
diff --git a/src/hotspot/share/gc/shared/cardTable.cpp b/src/hotspot/share/gc/shared/cardTable.cpp
index acd4bda6e10..53d98ba817e 100644
--- a/src/hotspot/share/gc/shared/cardTable.cpp
+++ b/src/hotspot/share/gc/shared/cardTable.cpp
@@ -44,7 +44,7 @@ uint CardTable::_card_size = 0;
uint CardTable::_card_size_in_words = 0;
void CardTable::initialize_card_size() {
- assert(UseG1GC || UseParallelGC || UseSerialGC,
+ assert(UseG1GC || UseParallelGC || UseSerialGC || UseShenandoahGC,
"Initialize card size should only be called by card based collectors.");
_card_size = GCCardSizeInBytes;
diff --git a/src/hotspot/share/gc/shared/gcConfiguration.cpp b/src/hotspot/share/gc/shared/gcConfiguration.cpp
index 824e119e696..ba977c75627 100644
--- a/src/hotspot/share/gc/shared/gcConfiguration.cpp
+++ b/src/hotspot/share/gc/shared/gcConfiguration.cpp
@@ -47,6 +47,11 @@ GCName GCConfiguration::young_collector() const {
}
if (UseShenandoahGC) {
+#if INCLUDE_SHENANDOAHGC
+ if (ShenandoahCardBarrier) {
+ return ShenandoahYoung;
+ }
+#endif
return NA;
}
@@ -67,6 +72,11 @@ if (UseZGC) {
}
if (UseShenandoahGC) {
+#if INCLUDE_SHENANDOAHGC
+ if (ShenandoahCardBarrier) {
+ return ShenandoahOld;
+ }
+#endif
return Shenandoah;
}
diff --git a/src/hotspot/share/gc/shared/gcName.hpp b/src/hotspot/share/gc/shared/gcName.hpp
index b9b87c231ca..642d734f673 100644
--- a/src/hotspot/share/gc/shared/gcName.hpp
+++ b/src/hotspot/share/gc/shared/gcName.hpp
@@ -38,6 +38,8 @@ enum GCName {
ZMinor,
ZMajor,
Shenandoah,
+ ShenandoahYoung,
+ ShenandoahOld,
NA,
GCNameEndSentinel
};
@@ -56,6 +58,8 @@ class GCNameHelper {
case ZMinor: return "ZGC Minor";
case ZMajor: return "ZGC Major";
case Shenandoah: return "Shenandoah";
+ case ShenandoahYoung: return "Shenandoah Young";
+ case ShenandoahOld: return "Shenandoah Old";
case NA: return "N/A";
default: ShouldNotReachHere(); return nullptr;
}
diff --git a/src/hotspot/share/gc/shared/locationPrinter.inline.hpp b/src/hotspot/share/gc/shared/locationPrinter.inline.hpp
index bb79bf80a5b..2820c1ad5b9 100644
--- a/src/hotspot/share/gc/shared/locationPrinter.inline.hpp
+++ b/src/hotspot/share/gc/shared/locationPrinter.inline.hpp
@@ -27,6 +27,7 @@
#include "gc/shared/locationPrinter.hpp"
+#include "memory/resourceArea.inline.hpp"
#include "oops/compressedOops.inline.hpp"
#include "oops/oopsHierarchy.hpp"
@@ -51,6 +52,7 @@ oop BlockLocationPrinter::base_oop_or_null(void* addr) {
template
bool BlockLocationPrinter::print_location(outputStream* st, void* addr) {
+ ResourceMark rm;
// Check if addr points into Java heap.
bool in_heap = CollectedHeapT::heap()->is_in(addr);
if (in_heap) {
diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
index 0e8b02d247e..ade0504b973 100644
--- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2018, 2024, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,6 +26,7 @@
#include "precompiled.hpp"
#include "c1/c1_IR.hpp"
#include "gc/shared/satbMarkQueue.hpp"
+#include "gc/shenandoah/mode/shenandoahMode.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
@@ -72,7 +74,6 @@ void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info,
__ load(gc_state_addr, flag_val);
// Create a mask to test if the marking bit is set.
- // TODO: can we directly test if bit is set?
LIR_Opr mask = LIR_OprFact::intConst(ShenandoahHeap::MARKING);
LIR_Opr mask_reg = gen->new_register(T_INT);
__ move(mask, mask_reg);
@@ -190,6 +191,16 @@ void ShenandoahBarrierSetC1::store_at_resolved(LIRAccess& access, LIR_Opr value)
}
}
BarrierSetC1::store_at_resolved(access, value);
+
+ if (ShenandoahCardBarrier && access.is_oop()) {
+ DecoratorSet decorators = access.decorators();
+ bool is_array = (decorators & IS_ARRAY) != 0;
+ bool on_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
+
+ bool precise = is_array || on_anonymous;
+ LIR_Opr post_addr = precise ? access.resolved_addr() : access.base().opr();
+ post_barrier(access, post_addr, value);
+ }
}
LIR_Opr ShenandoahBarrierSetC1::resolve_address(LIRAccess& access, bool resolve_in_register) {
@@ -288,3 +299,62 @@ void ShenandoahBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob)
false, &lrb_phantom_code_gen_cl);
}
}
+
+void ShenandoahBarrierSetC1::post_barrier(LIRAccess& access, LIR_Opr addr, LIR_Opr new_val) {
+ assert(ShenandoahCardBarrier, "Should have been checked by caller");
+
+ DecoratorSet decorators = access.decorators();
+ LIRGenerator* gen = access.gen();
+ bool in_heap = (decorators & IN_HEAP) != 0;
+ if (!in_heap) {
+ return;
+ }
+
+ BarrierSet* bs = BarrierSet::barrier_set();
+ ShenandoahBarrierSet* ctbs = barrier_set_cast(bs);
+ CardTable* ct = ctbs->card_table();
+ LIR_Const* card_table_base = new LIR_Const(ct->byte_map_base());
+ if (addr->is_address()) {
+ LIR_Address* address = addr->as_address_ptr();
+ // ptr cannot be an object because we use this barrier for array card marks
+ // and addr can point in the middle of an array.
+ LIR_Opr ptr = gen->new_pointer_register();
+ if (!address->index()->is_valid() && address->disp() == 0) {
+ __ move(address->base(), ptr);
+ } else {
+ assert(address->disp() != max_jint, "lea doesn't support patched addresses!");
+ __ leal(addr, ptr);
+ }
+ addr = ptr;
+ }
+ assert(addr->is_register(), "must be a register at this point");
+
+ LIR_Opr tmp = gen->new_pointer_register();
+ if (two_operand_lir_form) {
+ __ move(addr, tmp);
+ __ unsigned_shift_right(tmp, CardTable::card_shift(), tmp);
+ } else {
+ __ unsigned_shift_right(addr, CardTable::card_shift(), tmp);
+ }
+
+ LIR_Address* card_addr;
+ if (gen->can_inline_as_constant(card_table_base)) {
+ card_addr = new LIR_Address(tmp, card_table_base->as_jint(), T_BYTE);
+ } else {
+ card_addr = new LIR_Address(tmp, gen->load_constant(card_table_base), T_BYTE);
+ }
+
+ LIR_Opr dirty = LIR_OprFact::intConst(CardTable::dirty_card_val());
+ if (UseCondCardMark) {
+ LIR_Opr cur_value = gen->new_register(T_INT);
+ __ move(card_addr, cur_value);
+
+ LabelObj* L_already_dirty = new LabelObj();
+ __ cmp(lir_cond_equal, cur_value, dirty);
+ __ branch(lir_cond_equal, L_already_dirty->label());
+ __ move(dirty, card_addr);
+ __ branch_destination(L_already_dirty->label());
+ } else {
+ __ move(dirty, card_addr);
+ }
+}
diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp
index 98b2aad8871..6c7ef49080f 100644
--- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp
@@ -243,6 +243,8 @@ class ShenandoahBarrierSetC1 : public BarrierSetC1 {
virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value);
+ void post_barrier(LIRAccess& access, LIR_Opr addr, LIR_Opr new_val);
+
public:
virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob);
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
index 691c78cd024..b906ae2ca0b 100644
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2018, 2023, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,6 +27,7 @@
#include "classfile/javaClasses.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahCardTable.hpp"
#include "gc/shenandoah/shenandoahForwarding.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahRuntime.hpp"
@@ -432,6 +434,90 @@ void ShenandoahBarrierSetC2::insert_pre_barrier(GraphKit* kit, Node* base_oop, N
kit->final_sync(ideal);
}
+Node* ShenandoahBarrierSetC2::byte_map_base_node(GraphKit* kit) const {
+ BarrierSet* bs = BarrierSet::barrier_set();
+ ShenandoahBarrierSet* ctbs = barrier_set_cast(bs);
+ CardTable::CardValue* card_table_base = ctbs->card_table()->byte_map_base();
+ if (card_table_base != nullptr) {
+ return kit->makecon(TypeRawPtr::make((address)card_table_base));
+ } else {
+ return kit->null();
+ }
+}
+
+void ShenandoahBarrierSetC2::post_barrier(GraphKit* kit,
+ Node* ctl,
+ Node* oop_store,
+ Node* obj,
+ Node* adr,
+ uint adr_idx,
+ Node* val,
+ BasicType bt,
+ bool use_precise) const {
+ assert(ShenandoahCardBarrier, "Should have been checked by caller");
+
+ // No store check needed if we're storing a null.
+ if (val != nullptr && val->is_Con()) {
+ // must be either an oop or NULL
+ const Type* t = val->bottom_type();
+ if (t == TypePtr::NULL_PTR || t == Type::TOP)
+ return;
+ }
+
+ if (ReduceInitialCardMarks && obj == kit->just_allocated_object(kit->control())) {
+ // We can skip marks on a freshly-allocated object in Eden.
+ // Keep this code in sync with new_deferred_store_barrier() in runtime.cpp.
+ // That routine informs GC to take appropriate compensating steps,
+ // upon a slow-path allocation, so as to make this card-mark
+ // elision safe.
+ return;
+ }
+
+ if (!use_precise) {
+ // All card marks for a (non-array) instance are in one place:
+ adr = obj;
+ }
+ // (Else it's an array (or unknown), and we want more precise card marks.)
+ assert(adr != nullptr, "");
+
+ IdealKit ideal(kit, true);
+
+ // Convert the pointer to an int prior to doing math on it
+ Node* cast = __ CastPX(__ ctrl(), adr);
+
+ // Divide by card size
+ Node* card_offset = __ URShiftX( cast, __ ConI(CardTable::card_shift()) );
+
+ // Combine card table base and card offset
+ Node* card_adr = __ AddP(__ top(), byte_map_base_node(kit), card_offset );
+
+ // Get the alias_index for raw card-mark memory
+ int adr_type = Compile::AliasIdxRaw;
+ Node* zero = __ ConI(0); // Dirty card value
+
+ if (UseCondCardMark) {
+ // The classic GC reference write barrier is typically implemented
+ // as a store into the global card mark table. Unfortunately
+ // unconditional stores can result in false sharing and excessive
+ // coherence traffic as well as false transactional aborts.
+ // UseCondCardMark enables MP "polite" conditional card mark
+ // stores. In theory we could relax the load from ctrl() to
+ // no_ctrl, but that doesn't buy much latitude.
+ Node* card_val = __ load( __ ctrl(), card_adr, TypeInt::BYTE, T_BYTE, adr_type);
+ __ if_then(card_val, BoolTest::ne, zero);
+ }
+
+ // Smash zero into card
+ __ store(__ ctrl(), card_adr, zero, T_BYTE, adr_type, MemNode::unordered);
+
+ if (UseCondCardMark) {
+ __ end_if();
+ }
+
+ // Final sync IdealKit and GraphKit.
+ kit->final_sync(ideal);
+}
+
#undef __
const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_Type() {
@@ -499,8 +585,22 @@ Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue&
assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
static_cast(val.type()), nullptr /* pre_val */, access.type());
+
+ Node* result = BarrierSetC2::store_at_resolved(access, val);
+
+ if (ShenandoahCardBarrier) {
+ const bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
+ const bool is_array = (decorators & IS_ARRAY) != 0;
+ const bool use_precise = is_array || anonymous;
+ post_barrier(kit, kit->control(), access.raw_access(), access.base(),
+ adr, adr_idx, val.node(), access.type(), use_precise);
+ }
+ return result;
+ } else {
+ assert(access.is_opt_access(), "only for optimization passes");
+ assert(((decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");
+ return BarrierSetC2::store_at_resolved(access, val);
}
- return BarrierSetC2::store_at_resolved(access, val);
}
Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
@@ -571,7 +671,7 @@ Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val
}
Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
- Node* new_val, const Type* value_type) const {
+ Node* new_val, const Type* value_type) const {
GraphKit* kit = access.kit();
if (access.is_oop()) {
shenandoah_write_barrier_pre(kit, false /* do_load */,
@@ -612,6 +712,10 @@ Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess
}
#endif
load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(nullptr, load_store, access.decorators()));
+ if (ShenandoahCardBarrier) {
+ post_barrier(kit, kit->control(), access.raw_access(), access.base(),
+ access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true);
+ }
return load_store;
}
return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
@@ -666,6 +770,10 @@ Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAcces
}
access.set_raw_access(load_store);
pin_atomic_op(access);
+ if (ShenandoahCardBarrier) {
+ post_barrier(kit, kit->control(), access.raw_access(), access.base(),
+ access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true);
+ }
return load_store;
}
return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
@@ -679,6 +787,10 @@ Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& acces
shenandoah_write_barrier_pre(kit, false /* do_load */,
nullptr, nullptr, max_juint, nullptr, nullptr,
result /* pre_val */, T_OBJECT);
+ if (ShenandoahCardBarrier) {
+ post_barrier(kit, kit->control(), access.raw_access(), access.base(),
+ access.addr().node(), access.alias_idx(), val, T_OBJECT, true);
+ }
}
return result;
}
@@ -852,9 +964,25 @@ void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const
}
}
-void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
- if (is_shenandoah_wb_pre_call(n)) {
- shenandoah_eliminate_wb_pre(n, ¯o->igvn());
+void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
+ if (is_shenandoah_wb_pre_call(node)) {
+ shenandoah_eliminate_wb_pre(node, ¯o->igvn());
+ }
+ if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
+ Node* shift = node->unique_out();
+ Node* addp = shift->unique_out();
+ for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
+ Node* mem = addp->last_out(j);
+ if (UseCondCardMark && mem->is_Load()) {
+ assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
+ // The load is checking if the card has been written so
+ // replace it with zero to fold the test.
+ macro->replace_node(mem, macro->intcon(0));
+ continue;
+ }
+ assert(mem->is_Store(), "store required");
+ macro->replace_node(mem, mem->in(MemNode::Memory));
+ }
}
}
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp
index 6e241b39ce9..c1acde2118e 100644
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp
@@ -67,6 +67,18 @@ class ShenandoahBarrierSetC2 : public BarrierSetC2 {
Node* pre_val,
BasicType bt) const;
+ Node* byte_map_base_node(GraphKit* kit) const;
+
+ void post_barrier(GraphKit* kit,
+ Node* ctl,
+ Node* store,
+ Node* obj,
+ Node* adr,
+ uint adr_idx,
+ Node* val,
+ BasicType bt,
+ bool use_precise) const;
+
void insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset,
Node* pre_val, bool need_mem_bar) const;
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp
index 0bcb236ccbd..94c544a7ea3 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,13 +25,18 @@
#include "precompiled.hpp"
+#include "gc/shared/gcCause.hpp"
+#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp"
#include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.hpp"
+#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc/shenandoah/shenandoahFreeSet.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
#include "logging/log.hpp"
#include "logging/logTag.hpp"
+#include "runtime/globals.hpp"
#include "utilities/quickSort.hpp"
// These constants are used to adjust the margin of error for the moving
@@ -58,7 +64,8 @@ ShenandoahAdaptiveHeuristics::ShenandoahAdaptiveHeuristics(ShenandoahSpaceInfo*
ShenandoahHeuristics(space_info),
_margin_of_error_sd(ShenandoahAdaptiveInitialConfidence),
_spike_threshold_sd(ShenandoahAdaptiveInitialSpikeThreshold),
- _last_trigger(OTHER) { }
+ _last_trigger(OTHER),
+ _available(Moving_Average_Samples, ShenandoahAdaptiveDecayFactor) { }
ShenandoahAdaptiveHeuristics::~ShenandoahAdaptiveHeuristics() {}
@@ -90,7 +97,7 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand
size_t min_garbage = (free_target > actual_free ? (free_target - actual_free) : 0);
log_info(gc, ergo)("Adaptive CSet Selection. Target Free: " SIZE_FORMAT "%s, Actual Free: "
- SIZE_FORMAT "%s, Max CSet: " SIZE_FORMAT "%s, Min Garbage: " SIZE_FORMAT "%s",
+ SIZE_FORMAT "%s, Max Evacuation: " SIZE_FORMAT "%s, Min Garbage: " SIZE_FORMAT "%s",
byte_size_in_proper_unit(free_target), proper_unit_for_byte_size(free_target),
byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free),
byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset),
@@ -103,7 +110,7 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand
size_t cur_garbage = 0;
for (size_t idx = 0; idx < size; idx++) {
- ShenandoahHeapRegion* r = data[idx]._region;
+ ShenandoahHeapRegion* r = data[idx].get_region();
size_t new_cset = cur_cset + r->get_live_data_bytes();
size_t new_garbage = cur_garbage + r->garbage();
@@ -130,17 +137,19 @@ void ShenandoahAdaptiveHeuristics::record_success_concurrent() {
size_t available = _space_info->available();
- _available.add(available);
double z_score = 0.0;
- if (_available.sd() > 0) {
- z_score = (available - _available.avg()) / _available.sd();
+ double available_sd = _available.sd();
+ if (available_sd > 0) {
+ double available_avg = _available.avg();
+ z_score = (double(available) - available_avg) / available_sd;
+ log_debug(gc, ergo)("Available: " SIZE_FORMAT " %sB, z-score=%.3f. Average available: %.1f %sB +/- %.1f %sB.",
+ byte_size_in_proper_unit(available), proper_unit_for_byte_size(available),
+ z_score,
+ byte_size_in_proper_unit(available_avg), proper_unit_for_byte_size(available_avg),
+ byte_size_in_proper_unit(available_sd), proper_unit_for_byte_size(available_sd));
}
- log_debug(gc, ergo)("Available: " SIZE_FORMAT " %sB, z-score=%.3f. Average available: %.1f %sB +/- %.1f %sB.",
- byte_size_in_proper_unit(available), proper_unit_for_byte_size(available),
- z_score,
- byte_size_in_proper_unit(_available.avg()), proper_unit_for_byte_size(_available.avg()),
- byte_size_in_proper_unit(_available.sd()), proper_unit_for_byte_size(_available.sd()));
+ _available.add(double(available));
// In the case when a concurrent GC cycle completes successfully but with an
// unusually small amount of available memory we will adjust our trigger
@@ -195,42 +204,68 @@ static double saturate(double value, double min, double max) {
return MAX2(MIN2(value, max), min);
}
+// Rationale:
+// The idea is that there is an average allocation rate and there are occasional abnormal bursts (or spikes) of
+// allocations that exceed the average allocation rate. What do these spikes look like?
+//
+// 1. At certain phase changes, we may discard large amounts of data and replace it with large numbers of newly
+// allocated objects. This "spike" looks more like a phase change. We were in steady state at M bytes/sec
+// allocation rate and now we're in a "reinitialization phase" that looks like N bytes/sec. We need the "spike"
+// accommodation to give us enough runway to recalibrate our "average allocation rate".
+//
+// 2. The typical workload changes. "Suddenly", our typical workload of N TPS increases to N+delta TPS. This means
+// our average allocation rate needs to be adjusted. Once again, we need the "spike" accomodation to give us
+// enough runway to recalibrate our "average allocation rate".
+//
+// 3. Though there is an "average" allocation rate, a given workload's demand for allocation may be very bursty. We
+// allocate a bunch of LABs during the 5 ms that follow completion of a GC, then we perform no more allocations for
+// the next 150 ms. It seems we want the "spike" to represent the maximum divergence from average within the
+// period of time between consecutive evaluation of the should_start_gc() service. Here's the thinking:
+//
+// a) Between now and the next time I ask whether should_start_gc(), we might experience a spike representing
+// the anticipated burst of allocations. If that would put us over budget, then we should start GC immediately.
+// b) Between now and the anticipated depletion of allocation pool, there may be two or more bursts of allocations.
+// If there are more than one of these bursts, we can "approximate" that these will be separated by spans of
+// time with very little or no allocations so the "average" allocation rate should be a suitable approximation
+// of how this will behave.
+//
+// For cases 1 and 2, we need to "quickly" recalibrate the average allocation rate whenever we detect a change
+// in operation mode. We want some way to decide that the average rate has changed, while keeping average
+// allocation rate computation independent.
bool ShenandoahAdaptiveHeuristics::should_start_gc() {
- size_t max_capacity = _space_info->max_capacity();
size_t capacity = _space_info->soft_max_capacity();
- size_t available = _space_info->available();
+ size_t available = _space_info->soft_available();
size_t allocated = _space_info->bytes_allocated_since_gc_start();
- // Make sure the code below treats available without the soft tail.
- size_t soft_tail = max_capacity - capacity;
- available = (available > soft_tail) ? (available - soft_tail) : 0;
+ log_debug(gc)("should_start_gc? available: " SIZE_FORMAT ", soft_max_capacity: " SIZE_FORMAT
+ ", allocated: " SIZE_FORMAT, available, capacity, allocated);
// Track allocation rate even if we decide to start a cycle for other reasons.
double rate = _allocation_rate.sample(allocated);
_last_trigger = OTHER;
- size_t min_threshold = capacity / 100 * ShenandoahMinFreeThreshold;
+ size_t min_threshold = min_free_threshold();
if (available < min_threshold) {
- log_info(gc)("Trigger: Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)",
- byte_size_in_proper_unit(available), proper_unit_for_byte_size(available),
+ log_trigger("Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)",
+ byte_size_in_proper_unit(available), proper_unit_for_byte_size(available),
byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold));
return true;
}
+ // Check if we need to learn a bit about the application
const size_t max_learn = ShenandoahLearningSteps;
if (_gc_times_learned < max_learn) {
size_t init_threshold = capacity / 100 * ShenandoahInitFreeThreshold;
if (available < init_threshold) {
- log_info(gc)("Trigger: Learning " SIZE_FORMAT " of " SIZE_FORMAT ". Free (" SIZE_FORMAT "%s) is below initial threshold (" SIZE_FORMAT "%s)",
+ log_trigger("Learning " SIZE_FORMAT " of " SIZE_FORMAT ". Free (" SIZE_FORMAT "%s) is below initial threshold (" SIZE_FORMAT "%s)",
_gc_times_learned + 1, max_learn,
- byte_size_in_proper_unit(available), proper_unit_for_byte_size(available),
+ byte_size_in_proper_unit(available), proper_unit_for_byte_size(available),
byte_size_in_proper_unit(init_threshold), proper_unit_for_byte_size(init_threshold));
return true;
}
}
-
// Check if allocation headroom is still okay. This also factors in:
- // 1. Some space to absorb allocation spikes
+ // 1. Some space to absorb allocation spikes (ShenandoahAllocSpikeFactor)
// 2. Accumulated penalties from Degenerated and Full GC
size_t allocation_headroom = available;
@@ -240,28 +275,30 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() {
allocation_headroom -= MIN2(allocation_headroom, spike_headroom);
allocation_headroom -= MIN2(allocation_headroom, penalties);
- double avg_cycle_time = _gc_time_history->davg() + (_margin_of_error_sd * _gc_time_history->dsd());
+ double avg_cycle_time = _gc_cycle_time_history->davg() + (_margin_of_error_sd * _gc_cycle_time_history->dsd());
double avg_alloc_rate = _allocation_rate.upper_bound(_margin_of_error_sd);
+
+ log_debug(gc)("average GC time: %.2f ms, allocation rate: %.0f %s/s",
+ avg_cycle_time * 1000, byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate));
if (avg_cycle_time * avg_alloc_rate > allocation_headroom) {
- log_info(gc)("Trigger: Average GC time (%.2f ms) is above the time for average allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (margin of error = %.2f)",
+ log_trigger("Average GC time (%.2f ms) is above the time for average allocation rate (%.0f %sB/s)"
+ " to deplete free headroom (" SIZE_FORMAT "%s) (margin of error = %.2f)",
avg_cycle_time * 1000,
byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate),
byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom),
_margin_of_error_sd);
-
log_info(gc, ergo)("Free headroom: " SIZE_FORMAT "%s (free) - " SIZE_FORMAT "%s (spike) - " SIZE_FORMAT "%s (penalties) = " SIZE_FORMAT "%s",
byte_size_in_proper_unit(available), proper_unit_for_byte_size(available),
byte_size_in_proper_unit(spike_headroom), proper_unit_for_byte_size(spike_headroom),
byte_size_in_proper_unit(penalties), proper_unit_for_byte_size(penalties),
byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom));
-
_last_trigger = RATE;
return true;
}
bool is_spiking = _allocation_rate.is_spiking(rate, _spike_threshold_sd);
if (is_spiking && avg_cycle_time > allocation_headroom / rate) {
- log_info(gc)("Trigger: Average GC time (%.2f ms) is above the time for instantaneous allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (spike threshold = %.2f)",
+ log_trigger("Average GC time (%.2f ms) is above the time for instantaneous allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (spike threshold = %.2f)",
avg_cycle_time * 1000,
byte_size_in_proper_unit(rate), proper_unit_for_byte_size(rate),
byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom),
@@ -299,6 +336,13 @@ void ShenandoahAdaptiveHeuristics::adjust_spike_threshold(double amount) {
log_debug(gc, ergo)("Spike threshold now: %.2f", _spike_threshold_sd);
}
+size_t ShenandoahAdaptiveHeuristics::min_free_threshold() {
+ // Note that soft_max_capacity() / 100 * min_free_threshold is smaller than max_capacity() / 100 * min_free_threshold.
+ // We want to behave conservatively here, so use max_capacity(). By returning a larger value, we cause the GC to
+ // trigger when the remaining amount of free shrinks below the larger threshold.
+ return _space_info->max_capacity() / 100 * ShenandoahMinFreeThreshold;
+}
+
ShenandoahAllocationRate::ShenandoahAllocationRate() :
_last_sample_time(os::elapsedTime()),
_last_sample_value(0),
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp
index be86b7297b0..5ee10c6bebf 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,9 +26,10 @@
#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHADAPTIVEHEURISTICS_HPP
#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHADAPTIVEHEURISTICS_HPP
+#include "memory/allocation.hpp"
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
-#include "gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp"
#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
+#include "gc/shenandoah/shenandoahSharedVariables.hpp"
#include "utilities/numberSeq.hpp"
class ShenandoahAllocationRate : public CHeapObj {
@@ -39,7 +41,6 @@ class ShenandoahAllocationRate : public CHeapObj {
double upper_bound(double sds) const;
bool is_spiking(double rate, double threshold) const;
-
private:
double instantaneous_rate(double time, size_t allocated) const;
@@ -138,6 +139,12 @@ class ShenandoahAdaptiveHeuristics : public ShenandoahHeuristics {
// establishes what is 'normal' for the application and is used as a
// source of feedback to adjust trigger parameters.
TruncatedSeq _available;
+
+ // A conservative minimum threshold of free space that we'll try to maintain when possible.
+ // For example, we might trigger a concurrent gc if we are likely to drop below
+ // this threshold, or we might consider this when dynamically resizing generations
+ // in the generational case. Controlled by global flag ShenandoahMinFreeThreshold.
+ size_t min_free_threshold();
};
#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHADAPTIVEHEURISTICS_HPP
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp
index fa6b3e67fee..cdae9bb1285 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp
@@ -44,7 +44,7 @@ void ShenandoahAggressiveHeuristics::choose_collection_set_from_regiondata(Shena
RegionData* data, size_t size,
size_t free) {
for (size_t idx = 0; idx < size; idx++) {
- ShenandoahHeapRegion* r = data[idx]._region;
+ ShenandoahHeapRegion* r = data[idx].get_region();
if (r->garbage() > 0) {
cset->add_region(r);
}
@@ -52,7 +52,7 @@ void ShenandoahAggressiveHeuristics::choose_collection_set_from_regiondata(Shena
}
bool ShenandoahAggressiveHeuristics::should_start_gc() {
- log_info(gc)("Trigger: Start next cycle immediately");
+ log_trigger("Start next cycle immediately");
return true;
}
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp
index c8e882a0f64..2c7594e10dc 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp
@@ -58,17 +58,17 @@ bool ShenandoahCompactHeuristics::should_start_gc() {
size_t min_threshold = capacity / 100 * ShenandoahMinFreeThreshold;
if (available < min_threshold) {
- log_info(gc)("Trigger: Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)",
- byte_size_in_proper_unit(available), proper_unit_for_byte_size(available),
- byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold));
+ log_trigger("Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)",
+ byte_size_in_proper_unit(available), proper_unit_for_byte_size(available),
+ byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold));
return true;
}
size_t bytes_allocated = _space_info->bytes_allocated_since_gc_start();
if (bytes_allocated > threshold_bytes_allocated) {
- log_info(gc)("Trigger: Allocated since last cycle (" SIZE_FORMAT "%s) is larger than allocation threshold (" SIZE_FORMAT "%s)",
- byte_size_in_proper_unit(bytes_allocated), proper_unit_for_byte_size(bytes_allocated),
- byte_size_in_proper_unit(threshold_bytes_allocated), proper_unit_for_byte_size(threshold_bytes_allocated));
+ log_trigger("Allocated since last cycle (" SIZE_FORMAT "%s) is larger than allocation threshold (" SIZE_FORMAT "%s)",
+ byte_size_in_proper_unit(bytes_allocated), proper_unit_for_byte_size(bytes_allocated),
+ byte_size_in_proper_unit(threshold_bytes_allocated), proper_unit_for_byte_size(threshold_bytes_allocated));
return true;
}
@@ -89,7 +89,7 @@ void ShenandoahCompactHeuristics::choose_collection_set_from_regiondata(Shenando
size_t live_cset = 0;
for (size_t idx = 0; idx < size; idx++) {
- ShenandoahHeapRegion* r = data[idx]._region;
+ ShenandoahHeapRegion* r = data[idx].get_region();
size_t new_cset = live_cset + r->get_live_data_bytes();
if (new_cset < max_cset && r->garbage() > threshold) {
live_cset = new_cset;
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp
new file mode 100644
index 00000000000..5b6d82d97a4
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp
@@ -0,0 +1,290 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.hpp"
+#include "gc/shenandoah/shenandoahCollectionSet.hpp"
+#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
+#include "gc/shenandoah/shenandoahGeneration.hpp"
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
+#include "gc/shenandoah/shenandoahEvacInfo.hpp"
+#include "gc/shenandoah/shenandoahTrace.hpp"
+
+#include "logging/log.hpp"
+
+ShenandoahGenerationalHeuristics::ShenandoahGenerationalHeuristics(ShenandoahGeneration* generation)
+ : ShenandoahAdaptiveHeuristics(generation), _generation(generation) {
+}
+
+void ShenandoahGenerationalHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set) {
+ assert(collection_set->is_empty(), "Must be empty");
+
+ auto heap = ShenandoahGenerationalHeap::heap();
+ size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
+
+
+ // Check all pinned regions have updated status before choosing the collection set.
+ heap->assert_pinned_region_status();
+
+ // Step 1. Build up the region candidates we care about, rejecting losers and accepting winners right away.
+
+ size_t num_regions = heap->num_regions();
+
+ RegionData* candidates = _region_data;
+
+ size_t cand_idx = 0;
+ size_t preselected_candidates = 0;
+
+ size_t total_garbage = 0;
+
+ size_t immediate_garbage = 0;
+ size_t immediate_regions = 0;
+
+ size_t free = 0;
+ size_t free_regions = 0;
+
+ const uint tenuring_threshold = heap->age_census()->tenuring_threshold();
+
+ // This counts number of humongous regions that we intend to promote in this cycle.
+ size_t humongous_regions_promoted = 0;
+ // This counts number of regular regions that will be promoted in place.
+ size_t regular_regions_promoted_in_place = 0;
+ // This counts bytes of memory used by regular regions to be promoted in place.
+ size_t regular_regions_promoted_usage = 0;
+ // This counts bytes of memory free in regular regions to be promoted in place.
+ size_t regular_regions_promoted_free = 0;
+ // This counts bytes of garbage memory in regular regions to be promoted in place.
+ size_t regular_regions_promoted_garbage = 0;
+
+ for (size_t i = 0; i < num_regions; i++) {
+ ShenandoahHeapRegion* region = heap->get_region(i);
+ if (!_generation->contains(region)) {
+ continue;
+ }
+ size_t garbage = region->garbage();
+ total_garbage += garbage;
+ if (region->is_empty()) {
+ free_regions++;
+ free += region_size_bytes;
+ } else if (region->is_regular()) {
+ if (!region->has_live()) {
+ // We can recycle it right away and put it in the free set.
+ immediate_regions++;
+ immediate_garbage += garbage;
+ region->make_trash_immediate();
+ } else {
+ bool is_candidate;
+ // This is our candidate for later consideration.
+ if (collection_set->is_preselected(i)) {
+ assert(region->age() >= tenuring_threshold, "Preselection filter");
+ is_candidate = true;
+ preselected_candidates++;
+ // Set garbage value to maximum value to force this into the sorted collection set.
+ garbage = region_size_bytes;
+ } else if (region->is_young() && (region->age() >= tenuring_threshold)) {
+ // Note that for GLOBAL GC, region may be OLD, and OLD regions do not qualify for pre-selection
+
+ // This region is old enough to be promoted but it was not preselected, either because its garbage is below
+ // ShenandoahOldGarbageThreshold so it will be promoted in place, or because there is not sufficient room
+ // in old gen to hold the evacuated copies of this region's live data. In both cases, we choose not to
+ // place this region into the collection set.
+ if (region->get_top_before_promote() != nullptr) {
+ // Region was included for promotion-in-place
+ regular_regions_promoted_in_place++;
+ regular_regions_promoted_usage += region->used_before_promote();
+ regular_regions_promoted_free += region->free();
+ regular_regions_promoted_garbage += region->garbage();
+ }
+ is_candidate = false;
+ } else {
+ is_candidate = true;
+ }
+ if (is_candidate) {
+ candidates[cand_idx].set_region_and_garbage(region, garbage);
+ cand_idx++;
+ }
+ }
+ } else if (region->is_humongous_start()) {
+ // Reclaim humongous regions here, and count them as the immediate garbage
+#ifdef ASSERT
+ bool reg_live = region->has_live();
+ bool bm_live = heap->complete_marking_context()->is_marked(cast_to_oop(region->bottom()));
+ assert(reg_live == bm_live,
+ "Humongous liveness and marks should agree. Region live: %s; Bitmap live: %s; Region Live Words: " SIZE_FORMAT,
+ BOOL_TO_STR(reg_live), BOOL_TO_STR(bm_live), region->get_live_data_words());
+#endif
+ if (!region->has_live()) {
+ heap->trash_humongous_region_at(region);
+
+ // Count only the start. Continuations would be counted on "trash" path
+ immediate_regions++;
+ immediate_garbage += garbage;
+ } else {
+ if (region->is_young() && region->age() >= tenuring_threshold) {
+ oop obj = cast_to_oop(region->bottom());
+ size_t humongous_regions = ShenandoahHeapRegion::required_regions(obj->size() * HeapWordSize);
+ humongous_regions_promoted += humongous_regions;
+ }
+ }
+ } else if (region->is_trash()) {
+ // Count in just trashed collection set, during coalesced CM-with-UR
+ immediate_regions++;
+ immediate_garbage += garbage;
+ }
+ }
+ heap->old_generation()->set_expected_humongous_region_promotions(humongous_regions_promoted);
+ heap->old_generation()->set_expected_regular_region_promotions(regular_regions_promoted_in_place);
+ log_info(gc, ergo)("Planning to promote in place " SIZE_FORMAT " humongous regions and " SIZE_FORMAT
+ " regular regions, spanning a total of " SIZE_FORMAT " used bytes",
+ humongous_regions_promoted, regular_regions_promoted_in_place,
+ humongous_regions_promoted * ShenandoahHeapRegion::region_size_bytes() +
+ regular_regions_promoted_usage);
+
+ // Step 2. Look back at garbage statistics, and decide if we want to collect anything,
+ // given the amount of immediately reclaimable garbage. If we do, figure out the collection set.
+
+ assert (immediate_garbage <= total_garbage,
+ "Cannot have more immediate garbage than total garbage: " SIZE_FORMAT "%s vs " SIZE_FORMAT "%s",
+ byte_size_in_proper_unit(immediate_garbage), proper_unit_for_byte_size(immediate_garbage),
+ byte_size_in_proper_unit(total_garbage), proper_unit_for_byte_size(total_garbage));
+
+ size_t immediate_percent = (total_garbage == 0) ? 0 : (immediate_garbage * 100 / total_garbage);
+
+ bool doing_promote_in_place = (humongous_regions_promoted + regular_regions_promoted_in_place > 0);
+ if (doing_promote_in_place || (preselected_candidates > 0) || (immediate_percent <= ShenandoahImmediateThreshold)) {
+ // Only young collections need to prime the collection set.
+ if (_generation->is_young()) {
+ heap->old_generation()->heuristics()->prime_collection_set(collection_set);
+ }
+
+ // Call the subclasses to add young-gen regions into the collection set.
+ choose_collection_set_from_regiondata(collection_set, candidates, cand_idx, immediate_garbage + free);
+ }
+
+ if (collection_set->has_old_regions()) {
+ heap->shenandoah_policy()->record_mixed_cycle();
+ }
+
+ size_t cset_percent = (total_garbage == 0) ? 0 : (collection_set->garbage() * 100 / total_garbage);
+ size_t collectable_garbage = collection_set->garbage() + immediate_garbage;
+ size_t collectable_garbage_percent = (total_garbage == 0) ? 0 : (collectable_garbage * 100 / total_garbage);
+
+ log_info(gc, ergo)("Collectable Garbage: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%), "
+ "Immediate: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%), " SIZE_FORMAT " regions, "
+ "CSet: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%), " SIZE_FORMAT " regions",
+
+ byte_size_in_proper_unit(collectable_garbage),
+ proper_unit_for_byte_size(collectable_garbage),
+ collectable_garbage_percent,
+
+ byte_size_in_proper_unit(immediate_garbage),
+ proper_unit_for_byte_size(immediate_garbage),
+ immediate_percent,
+ immediate_regions,
+
+ byte_size_in_proper_unit(collection_set->garbage()),
+ proper_unit_for_byte_size(collection_set->garbage()),
+ cset_percent,
+ collection_set->count());
+
+ if (collection_set->garbage() > 0) {
+ size_t young_evac_bytes = collection_set->get_young_bytes_reserved_for_evacuation();
+ size_t promote_evac_bytes = collection_set->get_young_bytes_to_be_promoted();
+ size_t old_evac_bytes = collection_set->get_old_bytes_reserved_for_evacuation();
+ size_t total_evac_bytes = young_evac_bytes + promote_evac_bytes + old_evac_bytes;
+ log_info(gc, ergo)("Evacuation Targets: YOUNG: " SIZE_FORMAT "%s, "
+ "PROMOTE: " SIZE_FORMAT "%s, "
+ "OLD: " SIZE_FORMAT "%s, "
+ "TOTAL: " SIZE_FORMAT "%s",
+ byte_size_in_proper_unit(young_evac_bytes), proper_unit_for_byte_size(young_evac_bytes),
+ byte_size_in_proper_unit(promote_evac_bytes), proper_unit_for_byte_size(promote_evac_bytes),
+ byte_size_in_proper_unit(old_evac_bytes), proper_unit_for_byte_size(old_evac_bytes),
+ byte_size_in_proper_unit(total_evac_bytes), proper_unit_for_byte_size(total_evac_bytes));
+
+ ShenandoahEvacuationInformation evacInfo;
+ evacInfo.set_collection_set_regions(collection_set->count());
+ evacInfo.set_collection_set_used_before(collection_set->used());
+ evacInfo.set_collection_set_used_after(collection_set->live());
+ evacInfo.set_collected_old(old_evac_bytes);
+ evacInfo.set_collected_promoted(promote_evac_bytes);
+ evacInfo.set_collected_young(young_evac_bytes);
+ evacInfo.set_regions_promoted_humongous(humongous_regions_promoted);
+ evacInfo.set_regions_promoted_regular(regular_regions_promoted_in_place);
+ evacInfo.set_regular_promoted_garbage(regular_regions_promoted_garbage);
+ evacInfo.set_regular_promoted_free(regular_regions_promoted_free);
+ evacInfo.set_regions_immediate(immediate_regions);
+ evacInfo.set_immediate_size(immediate_garbage);
+ evacInfo.set_regions_freed(free_regions);
+
+ ShenandoahTracer().report_evacuation_info(&evacInfo);
+ }
+}
+
+
+size_t ShenandoahGenerationalHeuristics::add_preselected_regions_to_collection_set(ShenandoahCollectionSet* cset,
+ const RegionData* data,
+ size_t size) const {
+#ifdef ASSERT
+ const uint tenuring_threshold = ShenandoahGenerationalHeap::heap()->age_census()->tenuring_threshold();
+#endif
+
+ // cur_young_garbage represents the amount of memory to be reclaimed from young-gen. In the case that live objects
+ // are known to be promoted out of young-gen, we count this as cur_young_garbage because this memory is reclaimed
+ // from young-gen and becomes available to serve future young-gen allocation requests.
+ size_t cur_young_garbage = 0;
+ for (size_t idx = 0; idx < size; idx++) {
+ ShenandoahHeapRegion* r = data[idx].get_region();
+ if (cset->is_preselected(r->index())) {
+ assert(r->age() >= tenuring_threshold, "Preselected regions must have tenure age");
+ // Entire region will be promoted, This region does not impact young-gen or old-gen evacuation reserve.
+ // This region has been pre-selected and its impact on promotion reserve is already accounted for.
+
+ // r->used() is r->garbage() + r->get_live_data_bytes()
+ // Since all live data in this region is being evacuated from young-gen, it is as if this memory
+ // is garbage insofar as young-gen is concerned. Counting this as garbage reduces the need to
+ // reclaim highly utilized young-gen regions just for the sake of finding min_garbage to reclaim
+ // within young-gen memory.
+
+ cur_young_garbage += r->garbage();
+ cset->add_region(r);
+ }
+ }
+ return cur_young_garbage;
+}
+
+void ShenandoahGenerationalHeuristics::log_cset_composition(ShenandoahCollectionSet* cset) const {
+ size_t collected_old = cset->get_old_bytes_reserved_for_evacuation();
+ size_t collected_promoted = cset->get_young_bytes_to_be_promoted();
+ size_t collected_young = cset->get_young_bytes_reserved_for_evacuation();
+
+ log_info(gc, ergo)(
+ "Chosen CSet evacuates young: " SIZE_FORMAT "%s (of which at least: " SIZE_FORMAT "%s are to be promoted), "
+ "old: " SIZE_FORMAT "%s",
+ byte_size_in_proper_unit(collected_young), proper_unit_for_byte_size(collected_young),
+ byte_size_in_proper_unit(collected_promoted), proper_unit_for_byte_size(collected_promoted),
+ byte_size_in_proper_unit(collected_old), proper_unit_for_byte_size(collected_old));
+}
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.hpp
new file mode 100644
index 00000000000..6708c63f042
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.hpp
@@ -0,0 +1,59 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHGENERATIONALHEURISTICS_HPP
+#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHGENERATIONALHEURISTICS_HPP
+
+
+#include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp"
+
+class ShenandoahGeneration;
+
+/*
+ * This class serves as the base class for heuristics used to trigger and
+ * choose the collection sets for young and global collections. It leans
+ * heavily on the existing functionality of ShenandoahAdaptiveHeuristics.
+ *
+ * It differs from the base class primarily in that choosing the collection
+ * set is responsible for mixed collections and in-place promotions of tenured
+ * regions.
+ */
+class ShenandoahGenerationalHeuristics : public ShenandoahAdaptiveHeuristics {
+
+public:
+ explicit ShenandoahGenerationalHeuristics(ShenandoahGeneration* generation);
+
+ void choose_collection_set(ShenandoahCollectionSet* collection_set) override;
+protected:
+ ShenandoahGeneration* _generation;
+
+ size_t add_preselected_regions_to_collection_set(ShenandoahCollectionSet* cset,
+ const RegionData* data,
+ size_t size) const;
+
+ void log_cset_composition(ShenandoahCollectionSet* cset) const;
+};
+
+
+#endif //SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHGENERATIONALHEURISTICS_HPP
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp
new file mode 100644
index 00000000000..4c1e6b7bdff
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp
@@ -0,0 +1,147 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/shenandoah/heuristics/shenandoahGlobalHeuristics.hpp"
+#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
+#include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
+
+#include "utilities/quickSort.hpp"
+
+ShenandoahGlobalHeuristics::ShenandoahGlobalHeuristics(ShenandoahGlobalGeneration* generation)
+ : ShenandoahGenerationalHeuristics(generation) {
+}
+
+
+void ShenandoahGlobalHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+ RegionData* data, size_t size,
+ size_t actual_free) {
+ // Better select garbage-first regions
+ QuickSort::sort(data, (int) size, compare_by_garbage);
+
+ choose_global_collection_set(cset, data, size, actual_free, 0 /* cur_young_garbage */);
+
+ log_cset_composition(cset);
+}
+
+
+void ShenandoahGlobalHeuristics::choose_global_collection_set(ShenandoahCollectionSet* cset,
+ const ShenandoahHeuristics::RegionData* data,
+ size_t size, size_t actual_free,
+ size_t cur_young_garbage) const {
+ auto heap = ShenandoahGenerationalHeap::heap();
+ size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
+ size_t capacity = heap->young_generation()->max_capacity();
+ size_t garbage_threshold = region_size_bytes * ShenandoahGarbageThreshold / 100;
+ size_t ignore_threshold = region_size_bytes * ShenandoahIgnoreGarbageThreshold / 100;
+ const uint tenuring_threshold = heap->age_census()->tenuring_threshold();
+
+ size_t young_evac_reserve = heap->young_generation()->get_evacuation_reserve();
+ size_t old_evac_reserve = heap->old_generation()->get_evacuation_reserve();
+ size_t max_young_cset = (size_t) (young_evac_reserve / ShenandoahEvacWaste);
+ size_t young_cur_cset = 0;
+ size_t max_old_cset = (size_t) (old_evac_reserve / ShenandoahOldEvacWaste);
+ size_t old_cur_cset = 0;
+
+ // Figure out how many unaffiliated young regions are dedicated to mutator and to evacuator. Allow the young
+ // collector's unaffiliated regions to be transferred to old-gen if old-gen has more easily reclaimed garbage
+ // than young-gen. At the end of this cycle, any excess regions remaining in old-gen will be transferred back
+ // to young. Do not transfer the mutator's unaffiliated regions to old-gen. Those must remain available
+ // to the mutator as it needs to be able to consume this memory during concurrent GC.
+
+ size_t unaffiliated_young_regions = heap->young_generation()->free_unaffiliated_regions();
+ size_t unaffiliated_young_memory = unaffiliated_young_regions * region_size_bytes;
+
+ if (unaffiliated_young_memory > max_young_cset) {
+ size_t unaffiliated_mutator_memory = unaffiliated_young_memory - max_young_cset;
+ unaffiliated_young_memory -= unaffiliated_mutator_memory;
+ unaffiliated_young_regions = unaffiliated_young_memory / region_size_bytes; // round down
+ unaffiliated_young_memory = unaffiliated_young_regions * region_size_bytes;
+ }
+
+ // We'll affiliate these unaffiliated regions with either old or young, depending on need.
+ max_young_cset -= unaffiliated_young_memory;
+
+ // Keep track of how many regions we plan to transfer from young to old.
+ size_t regions_transferred_to_old = 0;
+
+ size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_young_cset;
+ size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0;
+
+ log_info(gc, ergo)("Adaptive CSet Selection for GLOBAL. Max Young Evacuation: " SIZE_FORMAT
+ "%s, Max Old Evacuation: " SIZE_FORMAT "%s, Actual Free: " SIZE_FORMAT "%s.",
+ byte_size_in_proper_unit(max_young_cset), proper_unit_for_byte_size(max_young_cset),
+ byte_size_in_proper_unit(max_old_cset), proper_unit_for_byte_size(max_old_cset),
+ byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free));
+
+ for (size_t idx = 0; idx < size; idx++) {
+ ShenandoahHeapRegion* r = data[idx].get_region();
+ assert(!cset->is_preselected(r->index()), "There should be no preselected regions during GLOBAL GC");
+ bool add_region = false;
+ if (r->is_old() || (r->age() >= tenuring_threshold)) {
+ size_t new_cset = old_cur_cset + r->get_live_data_bytes();
+ if ((r->garbage() > garbage_threshold)) {
+ while ((new_cset > max_old_cset) && (unaffiliated_young_regions > 0)) {
+ unaffiliated_young_regions--;
+ regions_transferred_to_old++;
+ max_old_cset += region_size_bytes / ShenandoahOldEvacWaste;
+ }
+ }
+ if ((new_cset <= max_old_cset) && (r->garbage() > garbage_threshold)) {
+ add_region = true;
+ old_cur_cset = new_cset;
+ }
+ } else {
+ assert(r->is_young() && (r->age() < tenuring_threshold), "DeMorgan's law (assuming r->is_affiliated)");
+ size_t new_cset = young_cur_cset + r->get_live_data_bytes();
+ size_t region_garbage = r->garbage();
+ size_t new_garbage = cur_young_garbage + region_garbage;
+ bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage);
+
+ if (add_regardless || (r->garbage() > garbage_threshold)) {
+ while ((new_cset > max_young_cset) && (unaffiliated_young_regions > 0)) {
+ unaffiliated_young_regions--;
+ max_young_cset += region_size_bytes / ShenandoahEvacWaste;
+ }
+ }
+ if ((new_cset <= max_young_cset) && (add_regardless || (region_garbage > garbage_threshold))) {
+ add_region = true;
+ young_cur_cset = new_cset;
+ cur_young_garbage = new_garbage;
+ }
+ }
+ if (add_region) {
+ cset->add_region(r);
+ }
+ }
+
+ if (regions_transferred_to_old > 0) {
+ heap->generation_sizer()->force_transfer_to_old(regions_transferred_to_old);
+ heap->young_generation()->set_evacuation_reserve(young_evac_reserve - regions_transferred_to_old * region_size_bytes);
+ heap->old_generation()->set_evacuation_reserve(old_evac_reserve + regions_transferred_to_old * region_size_bytes);
+ }
+}
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.hpp
new file mode 100644
index 00000000000..1f95f75c521
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.hpp
@@ -0,0 +1,54 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHGLOBALHEURISTICS_HPP
+#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHGLOBALHEURISTICS_HPP
+
+
+#include "gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.hpp"
+
+class ShenandoahGlobalGeneration;
+
+/*
+ * This is a specialization of the generational heuristics which is aware
+ * of old and young regions and respects the configured evacuation parameters
+ * for such regions during a global collection of a generational heap.
+ */
+class ShenandoahGlobalHeuristics : public ShenandoahGenerationalHeuristics {
+public:
+ ShenandoahGlobalHeuristics(ShenandoahGlobalGeneration* generation);
+
+ void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+ RegionData* data, size_t size,
+ size_t actual_free) override;
+
+private:
+ void choose_global_collection_set(ShenandoahCollectionSet* cset,
+ const ShenandoahHeuristics::RegionData* data,
+ size_t size, size_t actual_free,
+ size_t cur_young_garbage) const;
+};
+
+
+#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHGLOBALHEURISTICS_HPP
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp
index 2d5af892c80..c2ad809f43a 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,38 +25,44 @@
#include "precompiled.hpp"
#include "gc/shared/gcCause.hpp"
-#include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
-#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
#include "logging/log.hpp"
#include "logging/logTag.hpp"
#include "runtime/globals_extension.hpp"
+#include "utilities/quickSort.hpp"
+// sort by decreasing garbage (so most garbage comes first)
int ShenandoahHeuristics::compare_by_garbage(RegionData a, RegionData b) {
- if (a._garbage > b._garbage)
+ if (a.get_garbage() > b.get_garbage()) {
return -1;
- else if (a._garbage < b._garbage)
+ } else if (a.get_garbage() < b.get_garbage()) {
return 1;
- else return 0;
+ } else {
+ return 0;
+ }
}
ShenandoahHeuristics::ShenandoahHeuristics(ShenandoahSpaceInfo* space_info) :
_space_info(space_info),
_region_data(nullptr),
+ _guaranteed_gc_interval(0),
_cycle_start(os::elapsedTime()),
_last_cycle_end(0),
_gc_times_learned(0),
_gc_time_penalties(0),
- _gc_time_history(new TruncatedSeq(10, ShenandoahAdaptiveDecayFactor)),
+ _gc_cycle_time_history(new TruncatedSeq(Moving_Average_Samples, ShenandoahAdaptiveDecayFactor)),
_metaspace_oom()
{
size_t num_regions = ShenandoahHeap::heap()->num_regions();
assert(num_regions > 0, "Sanity");
_region_data = NEW_C_HEAP_ARRAY(RegionData, num_regions, mtGC);
+ for (size_t i = 0; i < num_regions; i++) {
+ _region_data[i].clear();
+ }
}
ShenandoahHeuristics::~ShenandoahHeuristics() {
@@ -63,7 +70,7 @@ ShenandoahHeuristics::~ShenandoahHeuristics() {
}
void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set) {
- assert(collection_set->count() == 0, "Must be empty");
+ assert(collection_set->is_empty(), "Must be empty");
ShenandoahHeap* heap = ShenandoahHeap::heap();
@@ -105,8 +112,7 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec
region->make_trash_immediate();
} else {
// This is our candidate for later consideration.
- candidates[cand_idx]._region = region;
- candidates[cand_idx]._garbage = garbage;
+ candidates[cand_idx].set_region_and_garbage(region, garbage);
cand_idx++;
}
} else if (region->is_humongous_start()) {
@@ -147,13 +153,12 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec
}
size_t cset_percent = (total_garbage == 0) ? 0 : (collection_set->garbage() * 100 / total_garbage);
-
size_t collectable_garbage = collection_set->garbage() + immediate_garbage;
size_t collectable_garbage_percent = (total_garbage == 0) ? 0 : (collectable_garbage * 100 / total_garbage);
log_info(gc, ergo)("Collectable Garbage: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%), "
- "Immediate: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%), "
- "CSet: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%)",
+ "Immediate: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%), " SIZE_FORMAT " regions, "
+ "CSet: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%), " SIZE_FORMAT " regions",
byte_size_in_proper_unit(collectable_garbage),
proper_unit_for_byte_size(collectable_garbage),
@@ -162,10 +167,12 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec
byte_size_in_proper_unit(immediate_garbage),
proper_unit_for_byte_size(immediate_garbage),
immediate_percent,
+ immediate_regions,
byte_size_in_proper_unit(collection_set->garbage()),
proper_unit_for_byte_size(collection_set->garbage()),
- cset_percent);
+ cset_percent,
+ collection_set->count());
}
void ShenandoahHeuristics::record_cycle_start() {
@@ -180,15 +187,15 @@ bool ShenandoahHeuristics::should_start_gc() {
// Perform GC to cleanup metaspace
if (has_metaspace_oom()) {
// Some of vmTestbase/metaspace tests depend on following line to count GC cycles
- log_info(gc)("Trigger: %s", GCCause::to_string(GCCause::_metadata_GC_threshold));
+ log_trigger("%s", GCCause::to_string(GCCause::_metadata_GC_threshold));
return true;
}
- if (ShenandoahGuaranteedGCInterval > 0) {
+ if (_guaranteed_gc_interval > 0) {
double last_time_ms = (os::elapsedTime() - _last_cycle_end) * 1000;
- if (last_time_ms > ShenandoahGuaranteedGCInterval) {
- log_info(gc)("Trigger: Time since last GC (%.0f ms) is larger than guaranteed interval (" UINTX_FORMAT " ms)",
- last_time_ms, ShenandoahGuaranteedGCInterval);
+ if (last_time_ms > _guaranteed_gc_interval) {
+ log_trigger("Time since last GC (%.0f ms) is larger than guaranteed interval (" UINTX_FORMAT " ms)",
+ last_time_ms, _guaranteed_gc_interval);
return true;
}
}
@@ -202,7 +209,7 @@ bool ShenandoahHeuristics::should_degenerate_cycle() {
void ShenandoahHeuristics::adjust_penalty(intx step) {
assert(0 <= _gc_time_penalties && _gc_time_penalties <= 100,
- "In range before adjustment: " INTX_FORMAT, _gc_time_penalties);
+ "In range before adjustment: " INTX_FORMAT, _gc_time_penalties);
intx new_val = _gc_time_penalties + step;
if (new_val < 0) {
@@ -214,11 +221,29 @@ void ShenandoahHeuristics::adjust_penalty(intx step) {
_gc_time_penalties = new_val;
assert(0 <= _gc_time_penalties && _gc_time_penalties <= 100,
- "In range after adjustment: " INTX_FORMAT, _gc_time_penalties);
+ "In range after adjustment: " INTX_FORMAT, _gc_time_penalties);
+}
+
+void ShenandoahHeuristics::log_trigger(const char* fmt, ...) {
+ LogTarget(Info, gc) lt;
+ if (lt.is_enabled()) {
+ ResourceMark rm;
+ LogStream ls(lt);
+ ls.print_raw("Trigger", 7);
+ if (ShenandoahHeap::heap()->mode()->is_generational()) {
+ ls.print(" (%s)", _space_info->name());
+ }
+ ls.print_raw(": ", 2);
+ va_list va;
+ va_start(va, fmt);
+ ls.vprint(fmt, va);
+ va_end(va);
+ ls.cr();
+ }
}
void ShenandoahHeuristics::record_success_concurrent() {
- _gc_time_history->add(time_since_last_gc());
+ _gc_cycle_time_history->add(elapsed_cycle_time());
_gc_times_learned++;
adjust_penalty(Concurrent_Adjust);
@@ -256,6 +281,6 @@ void ShenandoahHeuristics::initialize() {
// Nothing to do by default.
}
-double ShenandoahHeuristics::time_since_last_gc() const {
+double ShenandoahHeuristics::elapsed_cycle_time() const {
return os::elapsedTime() - _cycle_start;
}
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp
index c4bfaed400d..8bd5c7775c4 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,11 +27,10 @@
#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHHEURISTICS_HPP
#include "gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp"
-#include "gc/shenandoah/shenandoahHeap.hpp"
-#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
#include "gc/shenandoah/shenandoahSharedVariables.hpp"
#include "memory/allocation.hpp"
#include "runtime/globals_extension.hpp"
+#include "utilities/numberSeq.hpp"
#define SHENANDOAH_ERGO_DISABLE_FLAG(name) \
do { \
@@ -69,23 +69,92 @@ class ShenandoahHeuristics : public CHeapObj {
static const intx Degenerated_Penalty = 10; // how much to penalize average GC duration history on Degenerated GC
static const intx Full_Penalty = 20; // how much to penalize average GC duration history on Full GC
+#ifdef ASSERT
+ enum UnionTag {
+ is_uninitialized, is_garbage, is_live_data
+ };
+#endif
+
protected:
- typedef struct {
+ static const uint Moving_Average_Samples = 10; // Number of samples to store in moving averages
+
+ class RegionData {
+ private:
ShenandoahHeapRegion* _region;
- size_t _garbage;
- } RegionData;
+ union {
+ size_t _garbage; // Not used by old-gen heuristics.
+ size_t _live_data; // Only used for old-gen heuristics, which prioritizes retention of _live_data over garbage reclaim
+ } _region_union;
+#ifdef ASSERT
+ UnionTag _union_tag;
+#endif
+ public:
+
+ inline void clear() {
+ _region = nullptr;
+ _region_union._garbage = 0;
+#ifdef ASSERT
+ _union_tag = is_uninitialized;
+#endif
+ }
+
+ inline void set_region_and_garbage(ShenandoahHeapRegion* region, size_t garbage) {
+ _region = region;
+ _region_union._garbage = garbage;
+#ifdef ASSERT
+ _union_tag = is_garbage;
+#endif
+ }
+
+ inline void set_region_and_livedata(ShenandoahHeapRegion* region, size_t live) {
+ _region = region;
+ _region_union._live_data = live;
+#ifdef ASSERT
+ _union_tag = is_live_data;
+#endif
+ }
+
+ inline ShenandoahHeapRegion* get_region() const {
+ assert(_union_tag != is_uninitialized, "Cannot fetch region from uninitialized RegionData");
+ return _region;
+ }
+
+ inline size_t get_garbage() const {
+ assert(_union_tag == is_garbage, "Invalid union fetch");
+ return _region_union._garbage;
+ }
+
+ inline size_t get_livedata() const {
+ assert(_union_tag == is_live_data, "Invalid union fetch");
+ return _region_union._live_data;
+ }
+ };
// Source of information about the memory space managed by this heuristic
ShenandoahSpaceInfo* _space_info;
+ // Depending on generation mode, region data represents the results of the relevant
+ // most recently completed marking pass:
+ // - in GLOBAL mode, global marking pass
+ // - in OLD mode, old-gen marking pass
+ // - in YOUNG mode, young-gen marking pass
+ //
+ // Note that there is some redundancy represented in region data because
+ // each instance is an array large enough to hold all regions. However,
+ // any region in young-gen is not in old-gen. And any time we are
+ // making use of the GLOBAL data, there is no need to maintain the
+ // YOUNG or OLD data. Consider this redundancy of data structure to
+ // have negligible cost unless proven otherwise.
RegionData* _region_data;
+ size_t _guaranteed_gc_interval;
+
double _cycle_start;
double _last_cycle_end;
size_t _gc_times_learned;
intx _gc_time_penalties;
- TruncatedSeq* _gc_time_history;
+ TruncatedSeq* _gc_cycle_time_history;
// There may be many threads that contend to set this flag
ShenandoahSharedFlag _metaspace_oom;
@@ -106,6 +175,10 @@ class ShenandoahHeuristics : public CHeapObj {
void clear_metaspace_oom() { _metaspace_oom.unset(); }
bool has_metaspace_oom() const { return _metaspace_oom.is_set(); }
+ void set_guaranteed_gc_interval(size_t guaranteed_gc_interval) {
+ _guaranteed_gc_interval = guaranteed_gc_interval;
+ }
+
virtual void record_cycle_start();
virtual void record_cycle_end();
@@ -127,6 +200,9 @@ class ShenandoahHeuristics : public CHeapObj {
virtual void choose_collection_set(ShenandoahCollectionSet* collection_set);
virtual bool can_unload_classes();
+
+ // This indicates whether or not the current cycle should unload classes.
+ // It does NOT indicate that a cycle should be started.
virtual bool should_unload_classes();
virtual const char* name() = 0;
@@ -134,7 +210,10 @@ class ShenandoahHeuristics : public CHeapObj {
virtual bool is_experimental() = 0;
virtual void initialize();
- double time_since_last_gc() const;
+ double elapsed_cycle_time() const;
+
+ // Format prefix and emit log message indicating a GC cycle hs been triggered
+ void log_trigger(const char* fmt, ...) ATTRIBUTE_PRINTF(2, 3);
};
#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHHEURISTICS_HPP
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp
new file mode 100644
index 00000000000..abb2b7b266a
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp
@@ -0,0 +1,734 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp"
+#include "gc/shenandoah/shenandoahCollectionSet.hpp"
+#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
+#include "logging/log.hpp"
+#include "utilities/quickSort.hpp"
+
+uint ShenandoahOldHeuristics::NOT_FOUND = -1U;
+
+// sort by increasing live (so least live comes first)
+int ShenandoahOldHeuristics::compare_by_live(RegionData a, RegionData b) {
+ if (a.get_livedata() < b.get_livedata()) {
+ return -1;
+ } else if (a.get_livedata() > b.get_livedata()) {
+ return 1;
+ } else {
+ return 0;
+ }
+}
+
+// sort by increasing index
+int ShenandoahOldHeuristics::compare_by_index(RegionData a, RegionData b) {
+ if (a.get_region()->index() < b.get_region()->index()) {
+ return -1;
+ } else if (a.get_region()->index() > b.get_region()->index()) {
+ return 1;
+ } else {
+ // quicksort may compare to self during search for pivot
+ return 0;
+ }
+}
+
+ShenandoahOldHeuristics::ShenandoahOldHeuristics(ShenandoahOldGeneration* generation, ShenandoahGenerationalHeap* gen_heap) :
+ ShenandoahHeuristics(generation),
+ _heap(gen_heap),
+ _old_gen(generation),
+ _first_pinned_candidate(NOT_FOUND),
+ _last_old_collection_candidate(0),
+ _next_old_collection_candidate(0),
+ _last_old_region(0),
+ _live_bytes_in_unprocessed_candidates(0),
+ _old_generation(generation),
+ _cannot_expand_trigger(false),
+ _fragmentation_trigger(false),
+ _growth_trigger(false),
+ _fragmentation_density(0.0),
+ _fragmentation_first_old_region(0),
+ _fragmentation_last_old_region(0)
+{
+}
+
+bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* collection_set) {
+ if (unprocessed_old_collection_candidates() == 0) {
+ return false;
+ }
+
+ if (_old_generation->is_preparing_for_mark()) {
+ // We have unprocessed old collection candidates, but the heuristic has given up on evacuating them.
+ // This is most likely because they were _all_ pinned at the time of the last mixed evacuation (and
+ // this in turn is most likely because there are just one or two candidate regions remaining).
+ log_info(gc, ergo)("Remaining " UINT32_FORMAT " old regions are being coalesced and filled", unprocessed_old_collection_candidates());
+ return false;
+ }
+
+ _first_pinned_candidate = NOT_FOUND;
+
+ uint included_old_regions = 0;
+ size_t evacuated_old_bytes = 0;
+ size_t collected_old_bytes = 0;
+
+ // If a region is put into the collection set, then this region's free (not yet used) bytes are no longer
+ // "available" to hold the results of other evacuations. This may cause a decrease in the remaining amount
+ // of memory that can still be evacuated. We address this by reducing the evacuation budget by the amount
+ // of live memory in that region and by the amount of unallocated memory in that region if the evacuation
+ // budget is constrained by availability of free memory.
+ const size_t old_evacuation_reserve = _old_generation->get_evacuation_reserve();
+ const size_t old_evacuation_budget = (size_t) ((double) old_evacuation_reserve / ShenandoahOldEvacWaste);
+ size_t unfragmented_available = _old_generation->free_unaffiliated_regions() * ShenandoahHeapRegion::region_size_bytes();
+ size_t fragmented_available;
+ size_t excess_fragmented_available;
+
+ if (unfragmented_available > old_evacuation_budget) {
+ unfragmented_available = old_evacuation_budget;
+ fragmented_available = 0;
+ excess_fragmented_available = 0;
+ } else {
+ assert(_old_generation->available() >= old_evacuation_budget, "Cannot budget more than is available");
+ fragmented_available = _old_generation->available() - unfragmented_available;
+ assert(fragmented_available + unfragmented_available >= old_evacuation_budget, "Budgets do not add up");
+ if (fragmented_available + unfragmented_available > old_evacuation_budget) {
+ excess_fragmented_available = (fragmented_available + unfragmented_available) - old_evacuation_budget;
+ fragmented_available -= excess_fragmented_available;
+ }
+ }
+
+ size_t remaining_old_evacuation_budget = old_evacuation_budget;
+ log_debug(gc)("Choose old regions for mixed collection: old evacuation budget: " SIZE_FORMAT "%s, candidates: %u",
+ byte_size_in_proper_unit(old_evacuation_budget), proper_unit_for_byte_size(old_evacuation_budget),
+ unprocessed_old_collection_candidates());
+
+ size_t lost_evacuation_capacity = 0;
+
+ // The number of old-gen regions that were selected as candidates for collection at the end of the most recent old-gen
+ // concurrent marking phase and have not yet been collected is represented by unprocessed_old_collection_candidates().
+ // Candidate regions are ordered according to increasing amount of live data. If there is not sufficient room to
+ // evacuate region N, then there is no need to even consider evacuating region N+1.
+ while (unprocessed_old_collection_candidates() > 0) {
+ // Old collection candidates are sorted in order of decreasing garbage contained therein.
+ ShenandoahHeapRegion* r = next_old_collection_candidate();
+ if (r == nullptr) {
+ break;
+ }
+ assert(r->is_regular(), "There should be no humongous regions in the set of mixed-evac candidates");
+
+ // If region r is evacuated to fragmented memory (to free memory within a partially used region), then we need
+ // to decrease the capacity of the fragmented memory by the scaled loss.
+
+ size_t live_data_for_evacuation = r->get_live_data_bytes();
+ size_t lost_available = r->free();
+
+ if ((lost_available > 0) && (excess_fragmented_available > 0)) {
+ if (lost_available < excess_fragmented_available) {
+ excess_fragmented_available -= lost_available;
+ lost_evacuation_capacity -= lost_available;
+ lost_available = 0;
+ } else {
+ lost_available -= excess_fragmented_available;
+ lost_evacuation_capacity -= excess_fragmented_available;
+ excess_fragmented_available = 0;
+ }
+ }
+ size_t scaled_loss = (size_t) ((double) lost_available / ShenandoahOldEvacWaste);
+ if ((lost_available > 0) && (fragmented_available > 0)) {
+ if (scaled_loss + live_data_for_evacuation < fragmented_available) {
+ fragmented_available -= scaled_loss;
+ scaled_loss = 0;
+ } else {
+ // We will have to allocate this region's evacuation memory from unfragmented memory, so don't bother
+ // to decrement scaled_loss
+ }
+ }
+ if (scaled_loss > 0) {
+ // We were not able to account for the lost free memory within fragmented memory, so we need to take this
+ // allocation out of unfragmented memory. Unfragmented memory does not need to account for loss of free.
+ if (live_data_for_evacuation > unfragmented_available) {
+ // There is not room to evacuate this region or any that come after it in within the candidates array.
+ break;
+ } else {
+ unfragmented_available -= live_data_for_evacuation;
+ }
+ } else {
+ // Since scaled_loss == 0, we have accounted for the loss of free memory, so we can allocate from either
+ // fragmented or unfragmented available memory. Use up the fragmented memory budget first.
+ size_t evacuation_need = live_data_for_evacuation;
+
+ if (evacuation_need > fragmented_available) {
+ evacuation_need -= fragmented_available;
+ fragmented_available = 0;
+ } else {
+ fragmented_available -= evacuation_need;
+ evacuation_need = 0;
+ }
+ if (evacuation_need > unfragmented_available) {
+ // There is not room to evacuate this region or any that come after it in within the candidates array.
+ break;
+ } else {
+ unfragmented_available -= evacuation_need;
+ // dead code: evacuation_need == 0;
+ }
+ }
+ collection_set->add_region(r);
+ included_old_regions++;
+ evacuated_old_bytes += live_data_for_evacuation;
+ collected_old_bytes += r->garbage();
+ consume_old_collection_candidate();
+ }
+
+ if (_first_pinned_candidate != NOT_FOUND) {
+ // Need to deal with pinned regions
+ slide_pinned_regions_to_front();
+ }
+ decrease_unprocessed_old_collection_candidates_live_memory(evacuated_old_bytes);
+ if (included_old_regions > 0) {
+ log_info(gc, ergo)("Old-gen piggyback evac (" UINT32_FORMAT " regions, evacuating " PROPERFMT ", reclaiming: " PROPERFMT ")",
+ included_old_regions, PROPERFMTARGS(evacuated_old_bytes), PROPERFMTARGS(collected_old_bytes));
+ }
+
+ if (unprocessed_old_collection_candidates() == 0) {
+ // We have added the last of our collection candidates to a mixed collection.
+ // Any triggers that occurred during mixed evacuations may no longer be valid. They can retrigger if appropriate.
+ clear_triggers();
+
+ _old_generation->complete_mixed_evacuations();
+ } else if (included_old_regions == 0) {
+ // We have candidates, but none were included for evacuation - are they all pinned?
+ // or did we just not have enough room for any of them in this collection set?
+ // We don't want a region with a stuck pin to prevent subsequent old collections, so
+ // if they are all pinned we transition to a state that will allow us to make these uncollected
+ // (pinned) regions parsable.
+ if (all_candidates_are_pinned()) {
+ log_info(gc, ergo)("All candidate regions " UINT32_FORMAT " are pinned", unprocessed_old_collection_candidates());
+ _old_generation->abandon_mixed_evacuations();
+ } else {
+ log_info(gc, ergo)("No regions selected for mixed collection. "
+ "Old evacuation budget: " PROPERFMT ", Remaining evacuation budget: " PROPERFMT
+ ", Lost capacity: " PROPERFMT
+ ", Next candidate: " UINT32_FORMAT ", Last candidate: " UINT32_FORMAT,
+ PROPERFMTARGS(old_evacuation_reserve),
+ PROPERFMTARGS(remaining_old_evacuation_budget),
+ PROPERFMTARGS(lost_evacuation_capacity),
+ _next_old_collection_candidate, _last_old_collection_candidate);
+ }
+ }
+
+ return (included_old_regions > 0);
+}
+
+bool ShenandoahOldHeuristics::all_candidates_are_pinned() {
+#ifdef ASSERT
+ if (uint(os::random()) % 100 < ShenandoahCoalesceChance) {
+ return true;
+ }
+#endif
+
+ for (uint i = _next_old_collection_candidate; i < _last_old_collection_candidate; ++i) {
+ ShenandoahHeapRegion* region = _region_data[i].get_region();
+ if (!region->is_pinned()) {
+ return false;
+ }
+ }
+ return true;
+}
+
+void ShenandoahOldHeuristics::slide_pinned_regions_to_front() {
+ // Find the first unpinned region to the left of the next region that
+ // will be added to the collection set. These regions will have been
+ // added to the cset, so we can use them to hold pointers to regions
+ // that were pinned when the cset was chosen.
+ // [ r p r p p p r r ]
+ // ^ ^ ^
+ // | | | pointer to next region to add to a mixed collection is here.
+ // | | first r to the left should be in the collection set now.
+ // | first pinned region, we don't need to look past this
+ uint write_index = NOT_FOUND;
+ for (uint search = _next_old_collection_candidate - 1; search > _first_pinned_candidate; --search) {
+ ShenandoahHeapRegion* region = _region_data[search].get_region();
+ if (!region->is_pinned()) {
+ write_index = search;
+ assert(region->is_cset(), "Expected unpinned region to be added to the collection set.");
+ break;
+ }
+ }
+
+ // If we could not find an unpinned region, it means there are no slots available
+ // to move up the pinned regions. In this case, we just reset our next index in the
+ // hopes that some of these regions will become unpinned before the next mixed
+ // collection. We may want to bailout of here instead, as it should be quite
+ // rare to have so many pinned regions and may indicate something is wrong.
+ if (write_index == NOT_FOUND) {
+ assert(_first_pinned_candidate != NOT_FOUND, "Should only be here if there are pinned regions.");
+ _next_old_collection_candidate = _first_pinned_candidate;
+ return;
+ }
+
+ // Find pinned regions to the left and move their pointer into a slot
+ // that was pointing at a region that has been added to the cset (or was pointing
+ // to a pinned region that we've already moved up). We are done when the leftmost
+ // pinned region has been slid up.
+ // [ r p r x p p p r ]
+ // ^ ^
+ // | | next region for mixed collections
+ // | Write pointer is here. We know this region is already in the cset
+ // | so we can clobber it with the next pinned region we find.
+ for (int32_t search = (int32_t)write_index - 1; search >= (int32_t)_first_pinned_candidate; --search) {
+ RegionData& skipped = _region_data[search];
+ if (skipped.get_region()->is_pinned()) {
+ RegionData& available_slot = _region_data[write_index];
+ available_slot.set_region_and_livedata(skipped.get_region(), skipped.get_livedata());
+ --write_index;
+ }
+ }
+
+ // Update to read from the leftmost pinned region. Plus one here because we decremented
+ // the write index to hold the next found pinned region. We are just moving it back now
+ // to point to the first pinned region.
+ _next_old_collection_candidate = write_index + 1;
+}
+
+void ShenandoahOldHeuristics::prepare_for_old_collections() {
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+
+ const size_t num_regions = heap->num_regions();
+ size_t cand_idx = 0;
+ size_t immediate_garbage = 0;
+ size_t immediate_regions = 0;
+ size_t live_data = 0;
+
+ RegionData* candidates = _region_data;
+ for (size_t i = 0; i < num_regions; i++) {
+ ShenandoahHeapRegion* region = heap->get_region(i);
+ if (!region->is_old()) {
+ continue;
+ }
+
+ size_t garbage = region->garbage();
+ size_t live_bytes = region->get_live_data_bytes();
+ live_data += live_bytes;
+
+ if (region->is_regular() || region->is_regular_pinned()) {
+ // Only place regular or pinned regions with live data into the candidate set.
+ // Pinned regions cannot be evacuated, but we are not actually choosing candidates
+ // for the collection set here. That happens later during the next young GC cycle,
+ // by which time, the pinned region may no longer be pinned.
+ if (!region->has_live()) {
+ assert(!region->is_pinned(), "Pinned region should have live (pinned) objects.");
+ region->make_trash_immediate();
+ immediate_regions++;
+ immediate_garbage += garbage;
+ } else {
+ region->begin_preemptible_coalesce_and_fill();
+ candidates[cand_idx].set_region_and_livedata(region, live_bytes);
+ cand_idx++;
+ }
+ } else if (region->is_humongous_start()) {
+ // This will handle humongous start regions whether they are also pinned, or not.
+ // If they are pinned, we expect them to hold live data, so they will not be
+ // turned into immediate garbage.
+ if (!region->has_live()) {
+ assert(!region->is_pinned(), "Pinned region should have live (pinned) objects.");
+ // The humongous object is dead, we can just return this region and the continuations
+ // immediately to the freeset - no evacuations are necessary here. The continuations
+ // will be made into trash by this method, so they'll be skipped by the 'is_regular'
+ // check above, but we still need to count the start region.
+ immediate_regions++;
+ immediate_garbage += garbage;
+ size_t region_count = heap->trash_humongous_region_at(region);
+ log_debug(gc)("Trashed " SIZE_FORMAT " regions for humongous object.", region_count);
+ }
+ } else if (region->is_trash()) {
+ // Count humongous objects made into trash here.
+ immediate_regions++;
+ immediate_garbage += garbage;
+ }
+ }
+
+ _old_generation->set_live_bytes_after_last_mark(live_data);
+
+ // Unlike young, we are more interested in efficiently packing OLD-gen than in reclaiming garbage first. We sort by live-data.
+ // Some regular regions may have been promoted in place with no garbage but also with very little live data. When we "compact"
+ // old-gen, we want to pack these underutilized regions together so we can have more unaffiliated (unfragmented) free regions
+ // in old-gen.
+
+ QuickSort::sort(candidates, cand_idx, compare_by_live);
+
+ const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
+
+ // The convention is to collect regions that have more than this amount of garbage.
+ const size_t garbage_threshold = region_size_bytes * ShenandoahOldGarbageThreshold / 100;
+
+ // Enlightened interpretation: collect regions that have less than this amount of live.
+ const size_t live_threshold = region_size_bytes - garbage_threshold;
+
+ _last_old_region = (uint)cand_idx;
+ _last_old_collection_candidate = (uint)cand_idx;
+ _next_old_collection_candidate = 0;
+
+ size_t unfragmented = 0;
+ size_t candidates_garbage = 0;
+
+ for (size_t i = 0; i < cand_idx; i++) {
+ size_t live = candidates[i].get_livedata();
+ if (live > live_threshold) {
+ // Candidates are sorted in increasing order of live data, so no regions after this will be below the threshold.
+ _last_old_collection_candidate = (uint)i;
+ break;
+ }
+ ShenandoahHeapRegion* r = candidates[i].get_region();
+ size_t region_garbage = r->garbage();
+ size_t region_free = r->free();
+ candidates_garbage += region_garbage;
+ unfragmented += region_free;
+ }
+
+ // defrag_count represents regions that are placed into the old collection set in order to defragment the memory
+ // that we try to "reserve" for humongous allocations.
+ size_t defrag_count = 0;
+ size_t total_uncollected_old_regions = _last_old_region - _last_old_collection_candidate;
+
+ if (cand_idx > _last_old_collection_candidate) {
+ // Above, we have added into the set of mixed-evacuation candidates all old-gen regions for which the live memory
+ // that they contain is below a particular old-garbage threshold. Regions that were not selected for the collection
+ // set hold enough live memory that it is not considered efficient (by "garbage-first standards") to compact these
+ // at the current time.
+ //
+ // However, if any of these regions that were rejected from the collection set reside within areas of memory that
+ // might interfere with future humongous allocation requests, we will prioritize them for evacuation at this time.
+ // Humongous allocations target the bottom of the heap. We want old-gen regions to congregate at the top of the
+ // heap.
+ //
+ // Sort the regions that were initially rejected from the collection set in order of index. This allows us to
+ // focus our attention on the regions that have low index value (i.e. the old-gen regions at the bottom of the heap).
+ QuickSort::sort(candidates + _last_old_collection_candidate, cand_idx - _last_old_collection_candidate,
+ compare_by_index);
+
+ const size_t first_unselected_old_region = candidates[_last_old_collection_candidate].get_region()->index();
+ const size_t last_unselected_old_region = candidates[cand_idx - 1].get_region()->index();
+ size_t span_of_uncollected_regions = 1 + last_unselected_old_region - first_unselected_old_region;
+
+ // Add no more than 1/8 of the existing old-gen regions to the set of mixed evacuation candidates.
+ const int MAX_FRACTION_OF_HUMONGOUS_DEFRAG_REGIONS = 8;
+ const size_t bound_on_additional_regions = cand_idx / MAX_FRACTION_OF_HUMONGOUS_DEFRAG_REGIONS;
+
+ // The heuristic old_is_fragmented trigger may be seeking to achieve up to 75% density. Allow ourselves to overshoot
+ // that target (at 7/8) so we will not have to do another defragmenting old collection right away.
+ while ((defrag_count < bound_on_additional_regions) &&
+ (total_uncollected_old_regions < 7 * span_of_uncollected_regions / 8)) {
+ ShenandoahHeapRegion* r = candidates[_last_old_collection_candidate].get_region();
+ assert(r->is_regular() || r->is_regular_pinned(), "Region " SIZE_FORMAT " has wrong state for collection: %s",
+ r->index(), ShenandoahHeapRegion::region_state_to_string(r->state()));
+ const size_t region_garbage = r->garbage();
+ const size_t region_free = r->free();
+ candidates_garbage += region_garbage;
+ unfragmented += region_free;
+ defrag_count++;
+ _last_old_collection_candidate++;
+
+ // We now have one fewer uncollected regions, and our uncollected span shrinks because we have removed its first region.
+ total_uncollected_old_regions--;
+ span_of_uncollected_regions =
+ 1 + last_unselected_old_region - candidates[_last_old_collection_candidate].get_region()->index();
+ }
+ }
+
+ // Note that we do not coalesce and fill occupied humongous regions
+ // HR: humongous regions, RR: regular regions, CF: coalesce and fill regions
+ const size_t collectable_garbage = immediate_garbage + candidates_garbage;
+ const size_t old_candidates = _last_old_collection_candidate;
+ const size_t mixed_evac_live = old_candidates * region_size_bytes - (candidates_garbage + unfragmented);
+ set_unprocessed_old_collection_candidates_live_memory(mixed_evac_live);
+
+ log_info(gc, ergo)("Old-Gen Collectable Garbage: " PROPERFMT " consolidated with free: " PROPERFMT ", over " SIZE_FORMAT " regions",
+ PROPERFMTARGS(collectable_garbage), PROPERFMTARGS(unfragmented), old_candidates);
+ log_info(gc, ergo)("Old-Gen Immediate Garbage: " PROPERFMT " over " SIZE_FORMAT " regions",
+ PROPERFMTARGS(immediate_garbage), immediate_regions);
+ log_info(gc, ergo)("Old regions selected for defragmentation: " SIZE_FORMAT, defrag_count);
+ log_info(gc, ergo)("Old regions not selected: " SIZE_FORMAT, total_uncollected_old_regions);
+
+ if (unprocessed_old_collection_candidates() > 0) {
+ _old_generation->transition_to(ShenandoahOldGeneration::EVACUATING);
+ } else if (has_coalesce_and_fill_candidates()) {
+ _old_generation->transition_to(ShenandoahOldGeneration::FILLING);
+ } else {
+ _old_generation->transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
+ }
+}
+
+size_t ShenandoahOldHeuristics::unprocessed_old_collection_candidates_live_memory() const {
+ return _live_bytes_in_unprocessed_candidates;
+}
+
+void ShenandoahOldHeuristics::set_unprocessed_old_collection_candidates_live_memory(size_t initial_live) {
+ _live_bytes_in_unprocessed_candidates = initial_live;
+}
+
+void ShenandoahOldHeuristics::decrease_unprocessed_old_collection_candidates_live_memory(size_t evacuated_live) {
+ assert(evacuated_live <= _live_bytes_in_unprocessed_candidates, "Cannot evacuate more than was present");
+ _live_bytes_in_unprocessed_candidates -= evacuated_live;
+}
+
+// Used by unit test: test_shenandoahOldHeuristic.cpp
+uint ShenandoahOldHeuristics::last_old_collection_candidate_index() const {
+ return _last_old_collection_candidate;
+}
+
+uint ShenandoahOldHeuristics::unprocessed_old_collection_candidates() const {
+ return _last_old_collection_candidate - _next_old_collection_candidate;
+}
+
+ShenandoahHeapRegion* ShenandoahOldHeuristics::next_old_collection_candidate() {
+ while (_next_old_collection_candidate < _last_old_collection_candidate) {
+ ShenandoahHeapRegion* next = _region_data[_next_old_collection_candidate].get_region();
+ if (!next->is_pinned()) {
+ return next;
+ } else {
+ assert(next->is_pinned(), "sanity");
+ if (_first_pinned_candidate == NOT_FOUND) {
+ _first_pinned_candidate = _next_old_collection_candidate;
+ }
+ }
+
+ _next_old_collection_candidate++;
+ }
+ return nullptr;
+}
+
+void ShenandoahOldHeuristics::consume_old_collection_candidate() {
+ _next_old_collection_candidate++;
+}
+
+unsigned int ShenandoahOldHeuristics::get_coalesce_and_fill_candidates(ShenandoahHeapRegion** buffer) {
+ uint end = _last_old_region;
+ uint index = _next_old_collection_candidate;
+ while (index < end) {
+ *buffer++ = _region_data[index++].get_region();
+ }
+ return (_last_old_region - _next_old_collection_candidate);
+}
+
+void ShenandoahOldHeuristics::abandon_collection_candidates() {
+ _last_old_collection_candidate = 0;
+ _next_old_collection_candidate = 0;
+ _last_old_region = 0;
+}
+
+void ShenandoahOldHeuristics::record_cycle_end() {
+ this->ShenandoahHeuristics::record_cycle_end();
+ clear_triggers();
+}
+
+void ShenandoahOldHeuristics::clear_triggers() {
+ // Clear any triggers that were set during mixed evacuations. Conditions may be different now that this phase has finished.
+ _cannot_expand_trigger = false;
+ _fragmentation_trigger = false;
+ _growth_trigger = false;
+}
+
+// This triggers old-gen collection if the number of regions "dedicated" to old generation is much larger than
+// is required to represent the memory currently used within the old generation. This trigger looks specifically
+// at density of the old-gen spanned region. A different mechanism triggers old-gen GC if the total number of
+// old-gen regions (regardless of how close the regions are to one another) grows beyond an anticipated growth target.
+void ShenandoahOldHeuristics::set_trigger_if_old_is_fragmented(size_t first_old_region, size_t last_old_region,
+ size_t old_region_count, size_t num_regions) {
+ if (ShenandoahGenerationalHumongousReserve > 0) {
+ // Our intent is to pack old-gen memory into the highest-numbered regions of the heap. Count all memory
+ // above first_old_region as the "span" of old generation.
+ size_t old_region_span = (first_old_region <= last_old_region)? (num_regions - first_old_region): 0;
+ // Given that memory at the bottom of the heap is reserved to represent humongous objects, the number of
+ // regions that old_gen is "allowed" to consume is less than the total heap size. The restriction on allowed
+ // span is not strictly enforced. This is a heuristic designed to reduce the likelihood that a humongous
+ // allocation request will require a STW full GC.
+ size_t allowed_old_gen_span = num_regions - (ShenandoahGenerationalHumongousReserve * num_regions) / 100;
+
+ size_t old_available = _old_gen->available() / HeapWordSize;
+ size_t region_size_words = ShenandoahHeapRegion::region_size_words();
+ size_t old_unaffiliated_available = _old_gen->free_unaffiliated_regions() * region_size_words;
+ assert(old_available >= old_unaffiliated_available, "sanity");
+ size_t old_fragmented_available = old_available - old_unaffiliated_available;
+
+ size_t old_words_consumed = old_region_count * region_size_words - old_fragmented_available;
+ size_t old_words_spanned = old_region_span * region_size_words;
+ double old_density = ((double) old_words_consumed) / old_words_spanned;
+
+ double old_span_percent = ((double) old_region_span) / allowed_old_gen_span;
+ if (old_span_percent > 0.50) {
+ // Squaring old_span_percent in the denominator below allows more aggressive triggering when we are
+ // above desired maximum span and less aggressive triggering when we are far below the desired maximum span.
+ double old_span_percent_squared = old_span_percent * old_span_percent;
+ if (old_density / old_span_percent_squared < 0.75) {
+ // We trigger old defragmentation, for example, if:
+ // old_span_percent is 110% and old_density is below 90.8%, or
+ // old_span_percent is 100% and old_density is below 75.0%, or
+ // old_span_percent is 90% and old_density is below 60.8%, or
+ // old_span_percent is 80% and old_density is below 48.0%, or
+ // old_span_percent is 70% and old_density is below 36.8%, or
+ // old_span_percent is 60% and old_density is below 27.0%, or
+ // old_span_percent is 50% and old_density is below 18.8%.
+
+ // Set the fragmentation trigger and related attributes
+ _fragmentation_trigger = true;
+ _fragmentation_density = old_density;
+ _fragmentation_first_old_region = first_old_region;
+ _fragmentation_last_old_region = last_old_region;
+ }
+ }
+ }
+}
+
+void ShenandoahOldHeuristics::set_trigger_if_old_is_overgrown() {
+ size_t old_used = _old_gen->used() + _old_gen->get_humongous_waste();
+ size_t trigger_threshold = _old_gen->usage_trigger_threshold();
+ // Detects unsigned arithmetic underflow
+ assert(old_used <= _heap->capacity(),
+ "Old used (" SIZE_FORMAT ", " SIZE_FORMAT") must not be more than heap capacity (" SIZE_FORMAT ")",
+ _old_gen->used(), _old_gen->get_humongous_waste(), _heap->capacity());
+ if (old_used > trigger_threshold) {
+ _growth_trigger = true;
+ }
+}
+
+void ShenandoahOldHeuristics::evaluate_triggers(size_t first_old_region, size_t last_old_region,
+ size_t old_region_count, size_t num_regions) {
+ set_trigger_if_old_is_fragmented(first_old_region, last_old_region, old_region_count, num_regions);
+ set_trigger_if_old_is_overgrown();
+}
+
+bool ShenandoahOldHeuristics::should_start_gc() {
+ // Cannot start a new old-gen GC until previous one has finished.
+ //
+ // Future refinement: under certain circumstances, we might be more sophisticated about this choice.
+ // For example, we could choose to abandon the previous old collection before it has completed evacuations.
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+ if (!_old_generation->can_start_gc() || heap->collection_set()->has_old_regions()) {
+ return false;
+ }
+
+ if (_cannot_expand_trigger) {
+ const size_t old_gen_capacity = _old_generation->max_capacity();
+ const size_t heap_capacity = heap->capacity();
+ const double percent = percent_of(old_gen_capacity, heap_capacity);
+ log_trigger("Expansion failure, current size: " SIZE_FORMAT "%s which is %.1f%% of total heap size",
+ byte_size_in_proper_unit(old_gen_capacity), proper_unit_for_byte_size(old_gen_capacity), percent);
+ return true;
+ }
+
+ if (_fragmentation_trigger) {
+ const size_t used = _old_generation->used();
+ const size_t used_regions_size = _old_generation->used_regions_size();
+
+ // used_regions includes humongous regions
+ const size_t used_regions = _old_generation->used_regions();
+ assert(used_regions_size > used_regions, "Cannot have more used than used regions");
+
+ size_t first_old_region, last_old_region;
+ double density;
+ get_fragmentation_trigger_reason_for_log_message(density, first_old_region, last_old_region);
+ const size_t span_of_old_regions = (last_old_region >= first_old_region)? last_old_region + 1 - first_old_region: 0;
+ const size_t fragmented_free = used_regions_size - used;
+
+ log_trigger("Old has become fragmented: "
+ SIZE_FORMAT "%s available bytes spread between range spanned from "
+ SIZE_FORMAT " to " SIZE_FORMAT " (" SIZE_FORMAT "), density: %.1f%%",
+ byte_size_in_proper_unit(fragmented_free), proper_unit_for_byte_size(fragmented_free),
+ first_old_region, last_old_region, span_of_old_regions, density * 100);
+ return true;
+ }
+
+ if (_growth_trigger) {
+ // Growth may be falsely triggered during mixed evacuations, before the mixed-evacuation candidates have been
+ // evacuated. Before acting on a false trigger, we check to confirm the trigger condition is still satisfied.
+ const size_t current_usage = _old_generation->used() + _old_generation->get_humongous_waste();
+ const size_t trigger_threshold = _old_generation->usage_trigger_threshold();
+ const size_t heap_size = heap->capacity();
+ const size_t ignore_threshold = (ShenandoahIgnoreOldGrowthBelowPercentage * heap_size) / 100;
+ size_t consecutive_young_cycles;
+ if ((current_usage < ignore_threshold) &&
+ ((consecutive_young_cycles = heap->shenandoah_policy()->consecutive_young_gc_count())
+ < ShenandoahDoNotIgnoreGrowthAfterYoungCycles)) {
+ log_debug(gc)("Ignoring Trigger: Old has overgrown: usage (" SIZE_FORMAT "%s) is below threshold ("
+ SIZE_FORMAT "%s) after " SIZE_FORMAT " consecutive completed young GCs",
+ byte_size_in_proper_unit(current_usage), proper_unit_for_byte_size(current_usage),
+ byte_size_in_proper_unit(ignore_threshold), proper_unit_for_byte_size(ignore_threshold),
+ consecutive_young_cycles);
+ _growth_trigger = false;
+ } else if (current_usage > trigger_threshold) {
+ const size_t live_at_previous_old = _old_generation->get_live_bytes_after_last_mark();
+ const double percent_growth = percent_of(current_usage - live_at_previous_old, live_at_previous_old);
+ log_trigger("Old has overgrown, live at end of previous OLD marking: "
+ SIZE_FORMAT "%s, current usage: " SIZE_FORMAT "%s, percent growth: %.1f%%",
+ byte_size_in_proper_unit(live_at_previous_old), proper_unit_for_byte_size(live_at_previous_old),
+ byte_size_in_proper_unit(current_usage), proper_unit_for_byte_size(current_usage), percent_growth);
+ return true;
+ } else {
+ // Mixed evacuations have decreased current_usage such that old-growth trigger is no longer relevant.
+ _growth_trigger = false;
+ }
+ }
+
+ // Otherwise, defer to inherited heuristic for gc trigger.
+ return this->ShenandoahHeuristics::should_start_gc();
+}
+
+void ShenandoahOldHeuristics::record_success_concurrent() {
+ // Forget any triggers that occurred while OLD GC was ongoing. If we really need to start another, it will retrigger.
+ clear_triggers();
+ this->ShenandoahHeuristics::record_success_concurrent();
+}
+
+void ShenandoahOldHeuristics::record_success_degenerated() {
+ // Forget any triggers that occurred while OLD GC was ongoing. If we really need to start another, it will retrigger.
+ clear_triggers();
+ this->ShenandoahHeuristics::record_success_degenerated();
+}
+
+void ShenandoahOldHeuristics::record_success_full() {
+ // Forget any triggers that occurred while OLD GC was ongoing. If we really need to start another, it will retrigger.
+ clear_triggers();
+ this->ShenandoahHeuristics::record_success_full();
+}
+
+const char* ShenandoahOldHeuristics::name() {
+ return "Old";
+}
+
+bool ShenandoahOldHeuristics::is_diagnostic() {
+ return false;
+}
+
+bool ShenandoahOldHeuristics::is_experimental() {
+ return true;
+}
+
+void ShenandoahOldHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* set,
+ ShenandoahHeuristics::RegionData* data,
+ size_t data_size, size_t free) {
+ ShouldNotReachHere();
+}
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp
new file mode 100644
index 00000000000..d77380926b6
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp
@@ -0,0 +1,206 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHOLDHEURISTICS_HPP
+#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHOLDHEURISTICS_HPP
+
+
+#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
+
+class ShenandoahCollectionSet;
+class ShenandoahHeapRegion;
+class ShenandoahOldGeneration;
+
+/*
+ * This heuristic is responsible for choosing a set of candidates for inclusion
+ * in mixed collections. These candidates are chosen when marking of the old
+ * generation is complete. Note that this list of candidates may live through
+ * several mixed collections.
+ *
+ * This heuristic is also responsible for triggering old collections. It has its
+ * own collection of triggers to decide whether to start an old collection. It does
+ * _not_ use any of the functionality from the adaptive heuristics for triggers.
+ * It also does not use any of the functionality from the heuristics base classes
+ * to choose the collection set. For these reasons, it does not extend from
+ * ShenandoahGenerationalHeuristics.
+ */
+class ShenandoahOldHeuristics : public ShenandoahHeuristics {
+
+private:
+
+ static uint NOT_FOUND;
+
+ ShenandoahGenerationalHeap* _heap;
+ ShenandoahOldGeneration* _old_gen;
+
+ // After final marking of the old generation, this heuristic will select
+ // a set of candidate regions to be included in subsequent mixed collections.
+ // The regions are sorted into a `_region_data` array (declared in base
+ // class) in decreasing order of garbage. The heuristic will give priority
+ // to regions containing more garbage.
+
+ // The following members are used to keep track of which candidate regions
+ // have yet to be added to a mixed collection. There is also some special
+ // handling for pinned regions, described further below.
+
+ // Pinned regions may not be included in the collection set. Any old regions
+ // which were pinned at the time when old regions were added to the mixed
+ // collection will have been skipped. These regions are still contain garbage,
+ // so we want to include them at the start of the list of candidates for the
+ // _next_ mixed collection cycle. This variable is the index of the _first_
+ // old region which is pinned when the mixed collection set is formed.
+ uint _first_pinned_candidate;
+
+ // This is the index of the last region which is above the garbage threshold.
+ // No regions after this will be considered for inclusion in a mixed collection
+ // set.
+ uint _last_old_collection_candidate;
+
+ // This index points to the first candidate in line to be added to the mixed
+ // collection set. It is updated as regions are added to the collection set.
+ uint _next_old_collection_candidate;
+
+ // This is the last index in the array of old regions which were active at
+ // the end of old final mark.
+ uint _last_old_region;
+
+ // How much live data must be evacuated from within the unprocessed mixed evacuation candidates?
+ size_t _live_bytes_in_unprocessed_candidates;
+
+ // Keep a pointer to our generation that we can use without down casting a protected member from the base class.
+ ShenandoahOldGeneration* _old_generation;
+
+ // Flags are set when promotion failure is detected (by gc thread), and cleared when
+ // old generation collection begins (by control thread). Flags are set and cleared at safepoints.
+ bool _cannot_expand_trigger;
+ bool _fragmentation_trigger;
+ bool _growth_trigger;
+
+ // Motivation for a fragmentation_trigger
+ double _fragmentation_density;
+ size_t _fragmentation_first_old_region;
+ size_t _fragmentation_last_old_region;
+
+ // Compare by live is used to prioritize compaction of old-gen regions. With old-gen compaction, the goal is
+ // to tightly pack long-lived objects into available regions. In most cases, there has not been an accumulation
+ // of garbage within old-gen regions. The more likely opportunity will be to combine multiple sparsely populated
+ // old-gen regions which may have been promoted in place into a smaller number of densely packed old-gen regions.
+ // This improves subsequent allocation efficiency and reduces the likelihood of allocation failure (including
+ // humongous allocation failure) due to fragmentation of the available old-gen allocation pool
+ static int compare_by_live(RegionData a, RegionData b);
+
+ static int compare_by_index(RegionData a, RegionData b);
+
+ // Set the fragmentation trigger if old-gen memory has become fragmented.
+ void set_trigger_if_old_is_fragmented(size_t first_old_region, size_t last_old_region,
+ size_t old_region_count, size_t num_regions);
+
+ // Set the overgrowth trigger if old-gen memory has grown beyond a particular threshold.
+ void set_trigger_if_old_is_overgrown();
+
+ protected:
+ void choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, RegionData* data, size_t data_size, size_t free) override;
+
+public:
+ explicit ShenandoahOldHeuristics(ShenandoahOldGeneration* generation, ShenandoahGenerationalHeap* gen_heap);
+
+ // Prepare for evacuation of old-gen regions by capturing the mark results of a recently completed concurrent mark pass.
+ void prepare_for_old_collections();
+
+ // Return true iff the collection set is primed with at least one old-gen region.
+ bool prime_collection_set(ShenandoahCollectionSet* set);
+
+ // How many old-collection candidates have not yet been processed?
+ uint unprocessed_old_collection_candidates() const;
+
+ // How much live memory must be evacuated from within old-collection candidates that have not yet been processed?
+ size_t unprocessed_old_collection_candidates_live_memory() const;
+
+ void set_unprocessed_old_collection_candidates_live_memory(size_t initial_live);
+
+ void decrease_unprocessed_old_collection_candidates_live_memory(size_t evacuated_live);
+
+ // How many old or hidden collection candidates have not yet been processed?
+ uint last_old_collection_candidate_index() const;
+
+ // Return the next old-collection candidate in order of decreasing amounts of garbage. (We process most-garbage regions
+ // first.) This does not consume the candidate. If the candidate is selected for inclusion in a collection set, then
+ // the candidate is consumed by invoking consume_old_collection_candidate().
+ ShenandoahHeapRegion* next_old_collection_candidate();
+
+ // Adjust internal state to reflect that one fewer old-collection candidate remains to be processed.
+ void consume_old_collection_candidate();
+
+ // Fill in buffer with all the old-collection regions that were identified at the end of the most recent old-gen
+ // mark to require their unmarked objects to be coalesced and filled. The buffer array must have at least
+ // last_old_region_index() entries, or memory may be corrupted when this function overwrites the
+ // end of the array.
+ unsigned int get_coalesce_and_fill_candidates(ShenandoahHeapRegion** buffer);
+
+ // True if there are old regions that need to be filled.
+ bool has_coalesce_and_fill_candidates() const { return coalesce_and_fill_candidates_count() > 0; }
+
+ // Return the number of old regions that need to be filled.
+ size_t coalesce_and_fill_candidates_count() const { return _last_old_region - _next_old_collection_candidate; }
+
+ // If a GLOBAL gc occurs, it will collect the entire heap which invalidates any collection candidates being
+ // held by this heuristic for supplying mixed collections.
+ void abandon_collection_candidates();
+
+ void trigger_cannot_expand() { _cannot_expand_trigger = true; };
+
+ inline void get_fragmentation_trigger_reason_for_log_message(double &density, size_t &first_index, size_t &last_index) {
+ density = _fragmentation_density;
+ first_index = _fragmentation_first_old_region;
+ last_index = _fragmentation_last_old_region;
+ }
+
+ void clear_triggers();
+
+ // Check whether conditions merit the start of old GC. Set appropriate trigger if so.
+ void evaluate_triggers(size_t first_old_region, size_t last_old_region, size_t old_region_count, size_t num_regions);
+
+ void record_cycle_end() override;
+
+ bool should_start_gc() override;
+
+ void record_success_concurrent() override;
+
+ void record_success_degenerated() override;
+
+ void record_success_full() override;
+
+ const char* name() override;
+
+ bool is_diagnostic() override;
+
+ bool is_experimental() override;
+
+private:
+ void slide_pinned_regions_to_front();
+ bool all_candidates_are_pinned();
+};
+
+#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHOLDHEURISTICS_HPP
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp
index 2e6b3d46ebe..7c65482d8c4 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp
@@ -68,7 +68,7 @@ void ShenandoahPassiveHeuristics::choose_collection_set_from_regiondata(Shenando
size_t live_cset = 0;
for (size_t idx = 0; idx < size; idx++) {
- ShenandoahHeapRegion* r = data[idx]._region;
+ ShenandoahHeapRegion* r = data[idx].get_region();
size_t new_cset = live_cset + r->get_live_data_bytes();
if (new_cset < max_cset && r->garbage() > threshold) {
live_cset = new_cset;
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp
index 3a58196da3c..29b94e2f68f 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp
@@ -36,9 +36,12 @@
*/
class ShenandoahSpaceInfo {
public:
+ virtual const char* name() const = 0;
virtual size_t soft_max_capacity() const = 0;
virtual size_t max_capacity() const = 0;
+ virtual size_t soft_available() const = 0;
virtual size_t available() const = 0;
+ virtual size_t used() const = 0;
virtual size_t bytes_allocated_since_gc_start() const = 0;
};
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp
index ee59194feb6..db179d0a80a 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp
@@ -52,7 +52,7 @@ bool ShenandoahStaticHeuristics::should_start_gc() {
size_t threshold_available = capacity / 100 * ShenandoahMinFreeThreshold;
if (available < threshold_available) {
- log_info(gc)("Trigger: Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)",
+ log_trigger("Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)",
byte_size_in_proper_unit(available), proper_unit_for_byte_size(available),
byte_size_in_proper_unit(threshold_available), proper_unit_for_byte_size(threshold_available));
return true;
@@ -66,7 +66,7 @@ void ShenandoahStaticHeuristics::choose_collection_set_from_regiondata(Shenandoa
size_t threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahGarbageThreshold / 100;
for (size_t idx = 0; idx < size; idx++) {
- ShenandoahHeapRegion* r = data[idx]._region;
+ ShenandoahHeapRegion* r = data[idx].get_region();
if (r->garbage() > threshold) {
cset->add_region(r);
}
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.cpp
new file mode 100644
index 00000000000..ced406611b1
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.cpp
@@ -0,0 +1,224 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp"
+#include "gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp"
+#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
+#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
+
+#include "utilities/quickSort.hpp"
+
+ShenandoahYoungHeuristics::ShenandoahYoungHeuristics(ShenandoahYoungGeneration* generation)
+ : ShenandoahGenerationalHeuristics(generation) {
+}
+
+
+void ShenandoahYoungHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+ RegionData* data, size_t size,
+ size_t actual_free) {
+ // See comments in ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata():
+ // we do the same here, but with the following adjustments for generational mode:
+ //
+ // In generational mode, the sort order within the data array is not strictly descending amounts
+ // of garbage. In particular, regions that have reached tenure age will be sorted into this
+ // array before younger regions that typically contain more garbage. This is one reason why,
+ // for example, we continue examining regions even after rejecting a region that has
+ // more live data than we can evacuate.
+
+ // Better select garbage-first regions
+ QuickSort::sort(data, (int) size, compare_by_garbage);
+
+ size_t cur_young_garbage = add_preselected_regions_to_collection_set(cset, data, size);
+
+ choose_young_collection_set(cset, data, size, actual_free, cur_young_garbage);
+
+ log_cset_composition(cset);
+}
+
+void ShenandoahYoungHeuristics::choose_young_collection_set(ShenandoahCollectionSet* cset,
+ const RegionData* data,
+ size_t size, size_t actual_free,
+ size_t cur_young_garbage) const {
+
+ auto heap = ShenandoahGenerationalHeap::heap();
+
+ size_t capacity = heap->young_generation()->max_capacity();
+ size_t garbage_threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahGarbageThreshold / 100;
+ size_t ignore_threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahIgnoreGarbageThreshold / 100;
+ const uint tenuring_threshold = heap->age_census()->tenuring_threshold();
+
+ // This is young-gen collection or a mixed evacuation.
+ // If this is mixed evacuation, the old-gen candidate regions have already been added.
+ size_t max_cset = (size_t) (heap->young_generation()->get_evacuation_reserve() / ShenandoahEvacWaste);
+ size_t cur_cset = 0;
+ size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_cset;
+ size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0;
+
+
+ log_info(gc, ergo)(
+ "Adaptive CSet Selection for YOUNG. Max Evacuation: " SIZE_FORMAT "%s, Actual Free: " SIZE_FORMAT "%s.",
+ byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset),
+ byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free));
+
+ for (size_t idx = 0; idx < size; idx++) {
+ ShenandoahHeapRegion* r = data[idx].get_region();
+ if (cset->is_preselected(r->index())) {
+ continue;
+ }
+ if (r->age() < tenuring_threshold) {
+ size_t new_cset = cur_cset + r->get_live_data_bytes();
+ size_t region_garbage = r->garbage();
+ size_t new_garbage = cur_young_garbage + region_garbage;
+ bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage);
+ assert(r->is_young(), "Only young candidates expected in the data array");
+ if ((new_cset <= max_cset) && (add_regardless || (region_garbage > garbage_threshold))) {
+ cur_cset = new_cset;
+ cur_young_garbage = new_garbage;
+ cset->add_region(r);
+ }
+ }
+ // Note that we do not add aged regions if they were not pre-selected. The reason they were not preselected
+ // is because there is not sufficient room in old-gen to hold their to-be-promoted live objects or because
+ // they are to be promoted in place.
+ }
+}
+
+
+bool ShenandoahYoungHeuristics::should_start_gc() {
+ auto heap = ShenandoahGenerationalHeap::heap();
+ ShenandoahOldGeneration* old_generation = heap->old_generation();
+ ShenandoahOldHeuristics* old_heuristics = old_generation->heuristics();
+
+ // Checks that an old cycle has run for at least ShenandoahMinimumOldTimeMs before allowing a young cycle.
+ if (ShenandoahMinimumOldTimeMs > 0) {
+ if (old_generation->is_preparing_for_mark() || old_generation->is_concurrent_mark_in_progress()) {
+ size_t old_time_elapsed = size_t(old_heuristics->elapsed_cycle_time() * 1000);
+ if (old_time_elapsed < ShenandoahMinimumOldTimeMs) {
+ return false;
+ }
+ }
+ }
+
+ // inherited triggers have already decided to start a cycle, so no further evaluation is required
+ if (ShenandoahAdaptiveHeuristics::should_start_gc()) {
+ return true;
+ }
+
+ // Get through promotions and mixed evacuations as quickly as possible. These cycles sometimes require significantly
+ // more time than traditional young-generation cycles so start them up as soon as possible. This is a "mitigation"
+ // for the reality that old-gen and young-gen activities are not truly "concurrent". If there is old-gen work to
+ // be done, we start up the young-gen GC threads so they can do some of this old-gen work. As implemented, promotion
+ // gets priority over old-gen marking.
+ size_t promo_expedite_threshold = percent_of(heap->young_generation()->max_capacity(), ShenandoahExpeditePromotionsThreshold);
+ size_t promo_potential = old_generation->get_promotion_potential();
+ if (promo_potential > promo_expedite_threshold) {
+ // Detect unsigned arithmetic underflow
+ assert(promo_potential < heap->capacity(), "Sanity");
+ log_trigger("Expedite promotion of " PROPERFMT, PROPERFMTARGS(promo_potential));
+ return true;
+ }
+
+ size_t mixed_candidates = old_heuristics->unprocessed_old_collection_candidates();
+ if (mixed_candidates > ShenandoahExpediteMixedThreshold && !heap->is_concurrent_weak_root_in_progress()) {
+ // We need to run young GC in order to open up some free heap regions so we can finish mixed evacuations.
+ // If concurrent weak root processing is in progress, it means the old cycle has chosen mixed collection
+ // candidates, but has not completed. There is no point in trying to start the young cycle before the old
+ // cycle completes.
+ log_trigger("Expedite mixed evacuation of " SIZE_FORMAT " regions", mixed_candidates);
+ return true;
+ }
+
+ return false;
+}
+
+// Return a conservative estimate of how much memory can be allocated before we need to start GC. The estimate is based
+// on memory that is currently available within young generation plus all of the memory that will be added to the young
+// generation at the end of the current cycle (as represented by young_regions_to_be_reclaimed) and on the anticipated
+// amount of time required to perform a GC.
+size_t ShenandoahYoungHeuristics::bytes_of_allocation_runway_before_gc_trigger(size_t young_regions_to_be_reclaimed) {
+ size_t capacity = _space_info->max_capacity();
+ size_t usage = _space_info->used();
+ size_t available = (capacity > usage)? capacity - usage: 0;
+ size_t allocated = _space_info->bytes_allocated_since_gc_start();
+
+ size_t available_young_collected = ShenandoahHeap::heap()->collection_set()->get_young_available_bytes_collected();
+ size_t anticipated_available =
+ available + young_regions_to_be_reclaimed * ShenandoahHeapRegion::region_size_bytes() - available_young_collected;
+ size_t spike_headroom = capacity * ShenandoahAllocSpikeFactor / 100;
+ size_t penalties = capacity * _gc_time_penalties / 100;
+
+ double rate = _allocation_rate.sample(allocated);
+
+ // At what value of available, would avg and spike triggers occur?
+ // if allocation_headroom < avg_cycle_time * avg_alloc_rate, then we experience avg trigger
+ // if allocation_headroom < avg_cycle_time * rate, then we experience spike trigger if is_spiking
+ //
+ // allocation_headroom =
+ // 0, if penalties > available or if penalties + spike_headroom > available
+ // available - penalties - spike_headroom, otherwise
+ //
+ // so we trigger if available - penalties - spike_headroom < avg_cycle_time * avg_alloc_rate, which is to say
+ // available < avg_cycle_time * avg_alloc_rate + penalties + spike_headroom
+ // or if available < penalties + spike_headroom
+ //
+ // since avg_cycle_time * avg_alloc_rate > 0, the first test is sufficient to test both conditions
+ //
+ // thus, evac_slack_avg is MIN2(0, available - avg_cycle_time * avg_alloc_rate + penalties + spike_headroom)
+ //
+ // similarly, evac_slack_spiking is MIN2(0, available - avg_cycle_time * rate + penalties + spike_headroom)
+ // but evac_slack_spiking is only relevant if is_spiking, as defined below.
+
+ double avg_cycle_time = _gc_cycle_time_history->davg() + (_margin_of_error_sd * _gc_cycle_time_history->dsd());
+ double avg_alloc_rate = _allocation_rate.upper_bound(_margin_of_error_sd);
+ size_t evac_slack_avg;
+ if (anticipated_available > avg_cycle_time * avg_alloc_rate + penalties + spike_headroom) {
+ evac_slack_avg = anticipated_available - (avg_cycle_time * avg_alloc_rate + penalties + spike_headroom);
+ } else {
+ // we have no slack because it's already time to trigger
+ evac_slack_avg = 0;
+ }
+
+ bool is_spiking = _allocation_rate.is_spiking(rate, _spike_threshold_sd);
+ size_t evac_slack_spiking;
+ if (is_spiking) {
+ if (anticipated_available > avg_cycle_time * rate + penalties + spike_headroom) {
+ evac_slack_spiking = anticipated_available - (avg_cycle_time * rate + penalties + spike_headroom);
+ } else {
+ // we have no slack because it's already time to trigger
+ evac_slack_spiking = 0;
+ }
+ } else {
+ evac_slack_spiking = evac_slack_avg;
+ }
+
+ size_t threshold = min_free_threshold();
+ size_t evac_min_threshold = (anticipated_available > threshold)? anticipated_available - threshold: 0;
+ return MIN3(evac_slack_spiking, evac_slack_avg, evac_min_threshold);
+}
+
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp
new file mode 100644
index 00000000000..b9d64059680
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp
@@ -0,0 +1,57 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHYOUNGHEURISTICS_HPP
+#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHYOUNGHEURISTICS_HPP
+
+#include "gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.hpp"
+
+class ShenandoahYoungGeneration;
+
+/*
+ * This is a specialization of the generational heuristic which chooses
+ * young regions for evacuation. This heuristic also has additional triggers
+ * designed to expedite mixed collections and promotions.
+ */
+class ShenandoahYoungHeuristics : public ShenandoahGenerationalHeuristics {
+public:
+ explicit ShenandoahYoungHeuristics(ShenandoahYoungGeneration* generation);
+
+
+ void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+ RegionData* data, size_t size,
+ size_t actual_free) override;
+
+ bool should_start_gc() override;
+
+ size_t bytes_of_allocation_runway_before_gc_trigger(size_t young_regions_to_be_reclaimed);
+
+private:
+ void choose_young_collection_set(ShenandoahCollectionSet* cset,
+ const RegionData* data,
+ size_t size, size_t actual_free,
+ size_t cur_young_garbage) const;
+
+};
+
+#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHYOUNGHEURISTICS_HPP
diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp
new file mode 100644
index 00000000000..3a34a9aa6a8
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp
@@ -0,0 +1,64 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp"
+#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp"
+#include "logging/log.hpp"
+#include "logging/logTag.hpp"
+#include "runtime/globals_extension.hpp"
+
+void ShenandoahGenerationalMode::initialize_flags() const {
+
+#if !(defined AARCH64 || defined AMD64 || defined IA32 || defined PPC64 || defined RISCV64)
+ vm_exit_during_initialization("Shenandoah Generational GC is not supported on this platform.");
+#endif
+
+ // Exit if the user has asked ShenandoahCardBarrier to be disabled
+ if (!FLAG_IS_DEFAULT(ShenandoahCardBarrier)) {
+ SHENANDOAH_CHECK_FLAG_SET(ShenandoahCardBarrier);
+ }
+
+ // Enable card-marking post-write barrier for tracking old to young pointers
+ FLAG_SET_DEFAULT(ShenandoahCardBarrier, true);
+
+ if (ClassUnloading) {
+ FLAG_SET_DEFAULT(VerifyBeforeExit, false);
+ }
+
+ SHENANDOAH_ERGO_OVERRIDE_DEFAULT(GCTimeRatio, 70);
+ SHENANDOAH_ERGO_ENABLE_FLAG(ExplicitGCInvokesConcurrent);
+ SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent);
+
+ // This helps most multi-core hardware hosts, enable by default
+ SHENANDOAH_ERGO_ENABLE_FLAG(UseCondCardMark);
+
+ // Final configuration checks
+ SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier);
+ SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier);
+ SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier);
+ SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier);
+ SHENANDOAH_CHECK_FLAG_SET(ShenandoahCardBarrier);
+}
diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp
new file mode 100644
index 00000000000..0946858169a
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp
@@ -0,0 +1,39 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_MODE_SHENANDOAHGENERATIONALMODE_HPP
+#define SHARE_GC_SHENANDOAH_MODE_SHENANDOAHGENERATIONALMODE_HPP
+
+#include "gc/shenandoah/mode/shenandoahMode.hpp"
+
+class ShenandoahGenerationalMode : public ShenandoahMode {
+public:
+ virtual void initialize_flags() const;
+ virtual const char* name() { return "Generational"; }
+ virtual bool is_diagnostic() { return false; }
+ virtual bool is_experimental() { return true; }
+ virtual bool is_generational() { return true; }
+};
+
+#endif // SHARE_GC_SHENANDOAH_MODE_SHENANDOAHGENERATIONALMODE_HPP
diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp
new file mode 100644
index 00000000000..126062ab993
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2020, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp"
+#include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp"
+#include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp"
+#include "gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp"
+#include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp"
+#include "gc/shenandoah/mode/shenandoahMode.hpp"
+
+ShenandoahHeuristics* ShenandoahMode::initialize_heuristics(ShenandoahSpaceInfo* space_info) const {
+ if (ShenandoahGCHeuristics == nullptr) {
+ vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option (null)");
+ }
+
+ if (strcmp(ShenandoahGCHeuristics, "aggressive") == 0) {
+ return new ShenandoahAggressiveHeuristics(space_info);
+ } else if (strcmp(ShenandoahGCHeuristics, "static") == 0) {
+ return new ShenandoahStaticHeuristics(space_info);
+ } else if (strcmp(ShenandoahGCHeuristics, "adaptive") == 0) {
+ return new ShenandoahAdaptiveHeuristics(space_info);
+ } else if (strcmp(ShenandoahGCHeuristics, "compact") == 0) {
+ return new ShenandoahCompactHeuristics(space_info);
+ } else {
+ vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option");
+ }
+
+ ShouldNotReachHere();
+ return nullptr;
+}
+
diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp
index 5af6fa826d5..f3e98d92b2e 100644
--- a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp
+++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2019, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,8 +26,12 @@
#ifndef SHARE_GC_SHENANDOAH_MODE_SHENANDOAHMODE_HPP
#define SHARE_GC_SHENANDOAH_MODE_SHENANDOAHMODE_HPP
+#include "gc/shared/gc_globals.hpp"
#include "memory/allocation.hpp"
+#include "runtime/java.hpp"
+#include "utilities/formatBuffer.hpp"
+class ShenandoahSpaceInfo;
class ShenandoahHeuristics;
#define SHENANDOAH_CHECK_FLAG_SET(name) \
@@ -48,10 +53,11 @@ class ShenandoahHeuristics;
class ShenandoahMode : public CHeapObj {
public:
virtual void initialize_flags() const = 0;
- virtual ShenandoahHeuristics* initialize_heuristics() const = 0;
+ virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahSpaceInfo* space_info) const;
virtual const char* name() = 0;
virtual bool is_diagnostic() = 0;
virtual bool is_experimental() = 0;
+ virtual bool is_generational() { return false; }
};
#endif // SHARE_GC_SHENANDOAH_MODE_SHENANDOAHMODE_HPP
diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp
index 9b83df5424d..5c0ec2f884f 100644
--- a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp
+++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp
@@ -23,12 +23,13 @@
*/
#include "precompiled.hpp"
+#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp"
#include "gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp"
#include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "logging/log.hpp"
#include "logging/logTag.hpp"
-#include "runtime/globals_extension.hpp"
#include "runtime/java.hpp"
void ShenandoahPassiveMode::initialize_flags() const {
@@ -50,13 +51,17 @@ void ShenandoahPassiveMode::initialize_flags() const {
SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahCASBarrier);
SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahCloneBarrier);
SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahStackWatermarkBarrier);
+ SHENANDOAH_ERGO_DISABLE_FLAG(ShenandoahCardBarrier);
// Final configuration checks
- // No barriers are required to run.
+ // Passive mode does not instantiate the machinery to support the card table.
+ // Exit if the flag has been explicitly set.
+ SHENANDOAH_CHECK_FLAG_UNSET(ShenandoahCardBarrier);
}
-ShenandoahHeuristics* ShenandoahPassiveMode::initialize_heuristics() const {
+
+ShenandoahHeuristics* ShenandoahPassiveMode::initialize_heuristics(ShenandoahSpaceInfo* space_info) const {
if (ShenandoahGCHeuristics == nullptr) {
vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option (null)");
}
- return new ShenandoahPassiveHeuristics(ShenandoahHeap::heap());
+ return new ShenandoahPassiveHeuristics(space_info);
}
diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp
index c0e778174b3..032092a70ec 100644
--- a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp
+++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp
@@ -30,8 +30,7 @@
class ShenandoahPassiveMode : public ShenandoahMode {
public:
virtual void initialize_flags() const;
- virtual ShenandoahHeuristics* initialize_heuristics() const;
-
+ virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahSpaceInfo* space_info) const;
virtual const char* name() { return "Passive"; }
virtual bool is_diagnostic() { return true; }
virtual bool is_experimental() { return false; }
diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp
index c3ea11cf71e..f5023c7cae9 100644
--- a/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp
+++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp
@@ -23,12 +23,8 @@
*/
#include "precompiled.hpp"
-#include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp"
-#include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp"
-#include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp"
-#include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp"
+#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
#include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
-#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "logging/log.hpp"
#include "logging/logTag.hpp"
#include "runtime/globals_extension.hpp"
@@ -48,22 +44,5 @@ void ShenandoahSATBMode::initialize_flags() const {
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier);
SHENANDOAH_CHECK_FLAG_SET(ShenandoahStackWatermarkBarrier);
-}
-
-ShenandoahHeuristics* ShenandoahSATBMode::initialize_heuristics() const {
- if (ShenandoahGCHeuristics == nullptr) {
- vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option (null)");
- }
- ShenandoahHeap* heap = ShenandoahHeap::heap();
- if (strcmp(ShenandoahGCHeuristics, "aggressive") == 0) {
- return new ShenandoahAggressiveHeuristics(heap);
- } else if (strcmp(ShenandoahGCHeuristics, "static") == 0) {
- return new ShenandoahStaticHeuristics(heap);
- } else if (strcmp(ShenandoahGCHeuristics, "adaptive") == 0) {
- return new ShenandoahAdaptiveHeuristics(heap);
- } else if (strcmp(ShenandoahGCHeuristics, "compact") == 0) {
- return new ShenandoahCompactHeuristics(heap);
- }
- vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option");
- return nullptr;
+ SHENANDOAH_CHECK_FLAG_UNSET(ShenandoahCardBarrier);
}
diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.hpp
index f246f9b20c7..3c2ae5bde93 100644
--- a/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.hpp
+++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.hpp
@@ -32,7 +32,6 @@ class ShenandoahHeuristics;
class ShenandoahSATBMode : public ShenandoahMode {
public:
virtual void initialize_flags() const;
- virtual ShenandoahHeuristics* initialize_heuristics() const;
virtual const char* name() { return "Snapshot-At-The-Beginning (SATB)"; }
virtual bool is_diagnostic() { return false; }
virtual bool is_experimental() { return false; }
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAffiliation.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAffiliation.hpp
new file mode 100644
index 00000000000..6b3c846f5fe
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahAffiliation.hpp
@@ -0,0 +1,60 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHAFFILIATION_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHAFFILIATION_HPP
+
+enum ShenandoahAffiliation {
+ FREE,
+ YOUNG_GENERATION,
+ OLD_GENERATION,
+};
+
+inline const char* shenandoah_affiliation_code(ShenandoahAffiliation type) {
+ switch(type) {
+ case FREE:
+ return "F";
+ case YOUNG_GENERATION:
+ return "Y";
+ case OLD_GENERATION:
+ return "O";
+ default:
+ ShouldNotReachHere();
+ }
+}
+
+inline const char* shenandoah_affiliation_name(ShenandoahAffiliation type) {
+ switch (type) {
+ case FREE:
+ return "FREE";
+ case YOUNG_GENERATION:
+ return "YOUNG";
+ case OLD_GENERATION:
+ return "OLD";
+ default:
+ ShouldNotReachHere();
+ }
+}
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHAFFILIATION_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp
new file mode 100644
index 00000000000..7c574a9d0dd
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp
@@ -0,0 +1,383 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp"
+#include "gc/shenandoah/shenandoahAgeCensus.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+
+ShenandoahAgeCensus::ShenandoahAgeCensus() {
+ assert(ShenandoahHeap::heap()->mode()->is_generational(), "Only in generational mode");
+ if (ShenandoahGenerationalMinTenuringAge > ShenandoahGenerationalMaxTenuringAge) {
+ vm_exit_during_initialization(
+ err_msg("ShenandoahGenerationalMinTenuringAge=" SIZE_FORMAT
+ " should be no more than ShenandoahGenerationalMaxTenuringAge=" SIZE_FORMAT,
+ ShenandoahGenerationalMinTenuringAge, ShenandoahGenerationalMaxTenuringAge));
+ }
+
+ _global_age_table = NEW_C_HEAP_ARRAY(AgeTable*, MAX_SNAPSHOTS, mtGC);
+ CENSUS_NOISE(_global_noise = NEW_C_HEAP_ARRAY(ShenandoahNoiseStats, MAX_SNAPSHOTS, mtGC);)
+ _tenuring_threshold = NEW_C_HEAP_ARRAY(uint, MAX_SNAPSHOTS, mtGC);
+
+ for (int i = 0; i < MAX_SNAPSHOTS; i++) {
+ // Note that we don't now get perfdata from age_table
+ _global_age_table[i] = new AgeTable(false);
+ CENSUS_NOISE(_global_noise[i].clear();)
+ // Sentinel value
+ _tenuring_threshold[i] = MAX_COHORTS;
+ }
+ if (ShenandoahGenerationalAdaptiveTenuring && !ShenandoahGenerationalCensusAtEvac) {
+ size_t max_workers = ShenandoahHeap::heap()->max_workers();
+ _local_age_table = NEW_C_HEAP_ARRAY(AgeTable*, max_workers, mtGC);
+ CENSUS_NOISE(_local_noise = NEW_C_HEAP_ARRAY(ShenandoahNoiseStats, max_workers, mtGC);)
+ for (uint i = 0; i < max_workers; i++) {
+ _local_age_table[i] = new AgeTable(false);
+ CENSUS_NOISE(_local_noise[i].clear();)
+ }
+ } else {
+ _local_age_table = nullptr;
+ }
+ _epoch = MAX_SNAPSHOTS - 1; // see update_epoch()
+}
+
+CENSUS_NOISE(void ShenandoahAgeCensus::add(uint obj_age, uint region_age, uint region_youth, size_t size, uint worker_id) {)
+NO_CENSUS_NOISE(void ShenandoahAgeCensus::add(uint obj_age, uint region_age, size_t size, uint worker_id) {)
+ if (obj_age <= markWord::max_age) {
+ assert(obj_age < MAX_COHORTS && region_age < MAX_COHORTS, "Should have been tenured");
+#ifdef SHENANDOAH_CENSUS_NOISE
+ // Region ageing is stochastic and non-monotonic; this vitiates mortality
+ // demographics in ways that might defeat our algorithms. Marking may be a
+ // time when we might be able to correct this, but we currently do not do
+ // this. Like skipped statistics further below, we want to track the
+ // impact of this noise to see if this may be worthwhile. JDK-.
+ uint age = obj_age;
+ if (region_age > 0) {
+ add_aged(size, worker_id); // this tracking is coarse for now
+ age += region_age;
+ if (age >= MAX_COHORTS) {
+ age = (uint)(MAX_COHORTS - 1); // clamp
+ add_clamped(size, worker_id);
+ }
+ }
+ if (region_youth > 0) { // track object volume with retrograde age
+ add_young(size, worker_id);
+ }
+#else // SHENANDOAH_CENSUS_NOISE
+ uint age = MIN2(obj_age + region_age, (uint)(MAX_COHORTS - 1)); // clamp
+#endif // SHENANDOAH_CENSUS_NOISE
+ get_local_age_table(worker_id)->add(age, size);
+ } else {
+ // update skipped statistics
+ CENSUS_NOISE(add_skipped(size, worker_id);)
+ }
+}
+
+#ifdef SHENANDOAH_CENSUS_NOISE
+void ShenandoahAgeCensus::add_skipped(size_t size, uint worker_id) {
+ _local_noise[worker_id].skipped += size;
+}
+
+void ShenandoahAgeCensus::add_aged(size_t size, uint worker_id) {
+ _local_noise[worker_id].aged += size;
+}
+
+void ShenandoahAgeCensus::add_clamped(size_t size, uint worker_id) {
+ _local_noise[worker_id].clamped += size;
+}
+
+void ShenandoahAgeCensus::add_young(size_t size, uint worker_id) {
+ _local_noise[worker_id].young += size;
+}
+#endif // SHENANDOAH_CENSUS_NOISE
+
+// Prepare for a new census update, by clearing appropriate global slots.
+void ShenandoahAgeCensus::prepare_for_census_update() {
+ assert(_epoch < MAX_SNAPSHOTS, "Out of bounds");
+ if (++_epoch >= MAX_SNAPSHOTS) {
+ _epoch=0;
+ }
+ _global_age_table[_epoch]->clear();
+ CENSUS_NOISE(_global_noise[_epoch].clear();)
+}
+
+// Update the census data from appropriate sources,
+// and compute the new tenuring threshold.
+void ShenandoahAgeCensus::update_census(size_t age0_pop, AgeTable* pv1, AgeTable* pv2) {
+ prepare_for_census_update();
+ assert(_global_age_table[_epoch]->is_clear(), "Dirty decks");
+ CENSUS_NOISE(assert(_global_noise[_epoch].is_clear(), "Dirty decks");)
+ if (ShenandoahGenerationalAdaptiveTenuring && !ShenandoahGenerationalCensusAtEvac) {
+ assert(pv1 == nullptr && pv2 == nullptr, "Error, check caller");
+ // Seed cohort 0 with population that may have been missed during
+ // regular census.
+ _global_age_table[_epoch]->add((uint)0, age0_pop);
+
+ size_t max_workers = ShenandoahHeap::heap()->max_workers();
+ // Merge data from local age tables into the global age table for the epoch,
+ // clearing the local tables.
+ for (uint i = 0; i < max_workers; i++) {
+ // age stats
+ _global_age_table[_epoch]->merge(_local_age_table[i]);
+ _local_age_table[i]->clear(); // clear for next census
+ // Merge noise stats
+ CENSUS_NOISE(_global_noise[_epoch].merge(_local_noise[i]);)
+ CENSUS_NOISE(_local_noise[i].clear();)
+ }
+ } else {
+ // census during evac
+ assert(pv1 != nullptr && pv2 != nullptr, "Error, check caller");
+ _global_age_table[_epoch]->merge(pv1);
+ _global_age_table[_epoch]->merge(pv2);
+ }
+
+ update_tenuring_threshold();
+
+ // used for checking reasonableness of census coverage, non-product
+ // only.
+ NOT_PRODUCT(update_total();)
+}
+
+
+// Reset the epoch for the global age tables,
+// clearing all history.
+void ShenandoahAgeCensus::reset_global() {
+ assert(_epoch < MAX_SNAPSHOTS, "Out of bounds");
+ for (uint i = 0; i < MAX_SNAPSHOTS; i++) {
+ _global_age_table[i]->clear();
+ CENSUS_NOISE(_global_noise[i].clear();)
+ }
+ _epoch = MAX_SNAPSHOTS;
+ assert(_epoch < MAX_SNAPSHOTS, "Error");
+}
+
+// Reset the local age tables, clearing any partial census.
+void ShenandoahAgeCensus::reset_local() {
+ if (!ShenandoahGenerationalAdaptiveTenuring || ShenandoahGenerationalCensusAtEvac) {
+ assert(_local_age_table == nullptr, "Error");
+ return;
+ }
+ size_t max_workers = ShenandoahHeap::heap()->max_workers();
+ for (uint i = 0; i < max_workers; i++) {
+ _local_age_table[i]->clear();
+ CENSUS_NOISE(_local_noise[i].clear();)
+ }
+}
+
+#ifndef PRODUCT
+// Is global census information clear?
+bool ShenandoahAgeCensus::is_clear_global() {
+ assert(_epoch < MAX_SNAPSHOTS, "Out of bounds");
+ for (uint i = 0; i < MAX_SNAPSHOTS; i++) {
+ bool clear = _global_age_table[i]->is_clear();
+ CENSUS_NOISE(clear |= _global_noise[i].is_clear();)
+ if (!clear) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// Is local census information clear?
+bool ShenandoahAgeCensus::is_clear_local() {
+ if (!ShenandoahGenerationalAdaptiveTenuring || ShenandoahGenerationalCensusAtEvac) {
+ assert(_local_age_table == nullptr, "Error");
+ return true;
+ }
+ size_t max_workers = ShenandoahHeap::heap()->max_workers();
+ for (uint i = 0; i < max_workers; i++) {
+ bool clear = _local_age_table[i]->is_clear();
+ CENSUS_NOISE(clear |= _local_noise[i].is_clear();)
+ if (!clear) {
+ return false;
+ }
+ }
+ return true;
+}
+
+size_t ShenandoahAgeCensus::get_all_ages(uint snap) {
+ assert(snap < MAX_SNAPSHOTS, "Out of bounds");
+ size_t pop = 0;
+ const AgeTable* pv = _global_age_table[snap];
+ for (uint i = 0; i < MAX_COHORTS; i++) {
+ pop += pv->sizes[i];
+ }
+ return pop;
+}
+
+size_t ShenandoahAgeCensus::get_skipped(uint snap) {
+ assert(snap < MAX_SNAPSHOTS, "Out of bounds");
+ return _global_noise[snap].skipped;
+}
+
+void ShenandoahAgeCensus::update_total() {
+ _counted = get_all_ages(_epoch);
+ _skipped = get_skipped(_epoch);
+ _total = _counted + _skipped;
+}
+#endif // !PRODUCT
+
+void ShenandoahAgeCensus::update_tenuring_threshold() {
+ if (!ShenandoahGenerationalAdaptiveTenuring) {
+ _tenuring_threshold[_epoch] = InitialTenuringThreshold;
+ } else {
+ uint tt = compute_tenuring_threshold();
+ assert(tt <= MAX_COHORTS, "Out of bounds");
+ _tenuring_threshold[_epoch] = tt;
+ }
+ print();
+ log_trace(gc, age)("New tenuring threshold " UINTX_FORMAT " (min " UINTX_FORMAT ", max " UINTX_FORMAT")",
+ (uintx) _tenuring_threshold[_epoch], ShenandoahGenerationalMinTenuringAge, ShenandoahGenerationalMaxTenuringAge);
+}
+
+// Currently Shenandoah{Min,Max}TenuringAge have a floor of 1 because we
+// aren't set up to promote age 0 objects.
+uint ShenandoahAgeCensus::compute_tenuring_threshold() {
+ // Dispose of the extremal cases early so the loop below
+ // is less fragile.
+ if (ShenandoahGenerationalMaxTenuringAge == ShenandoahGenerationalMinTenuringAge) {
+ return ShenandoahGenerationalMaxTenuringAge; // Any value in [1,16]
+ }
+ assert(ShenandoahGenerationalMinTenuringAge < ShenandoahGenerationalMaxTenuringAge, "Error");
+
+ // Starting with the oldest cohort with a non-trivial population
+ // (as specified by ShenandoahGenerationalTenuringCohortPopulationThreshold) in the
+ // previous epoch, and working down the cohorts by age, find the
+ // oldest age that has a significant mortality rate (as specified by
+ // ShenandoahGenerationalTenuringMortalityRateThreshold). We use this as
+ // tenuring age to be used for the evacuation cycle to follow.
+ // Results are clamped between user-specified min & max guardrails,
+ // so we ignore any cohorts outside ShenandoahGenerational[Min,Max]Age.
+
+ // Current and previous epoch in ring
+ const uint cur_epoch = _epoch;
+ const uint prev_epoch = cur_epoch > 0 ? cur_epoch - 1 : markWord::max_age;
+
+ // Current and previous population vectors in ring
+ const AgeTable* cur_pv = _global_age_table[cur_epoch];
+ const AgeTable* prev_pv = _global_age_table[prev_epoch];
+ uint upper_bound = ShenandoahGenerationalMaxTenuringAge;
+ const uint prev_tt = previous_tenuring_threshold();
+ if (ShenandoahGenerationalCensusIgnoreOlderCohorts && prev_tt > 0) {
+ // We stay below the computed tenuring threshold for the last cycle plus 1,
+ // ignoring the mortality rates of any older cohorts.
+ upper_bound = MIN2(upper_bound, prev_tt + 1);
+ }
+ upper_bound = MIN2(upper_bound, markWord::max_age);
+
+ const uint lower_bound = MAX2((uint)ShenandoahGenerationalMinTenuringAge, (uint)1);
+
+ uint tenuring_threshold = upper_bound;
+ for (uint i = upper_bound; i >= lower_bound; i--) {
+ assert(i > 0, "Index (i-1) would underflow/wrap");
+ assert(i <= markWord::max_age, "Index i would overflow");
+ // Cohort of current age i
+ const size_t cur_pop = cur_pv->sizes[i];
+ const size_t prev_pop = prev_pv->sizes[i-1];
+ const double mr = mortality_rate(prev_pop, cur_pop);
+ if (prev_pop > ShenandoahGenerationalTenuringCohortPopulationThreshold &&
+ mr > ShenandoahGenerationalTenuringMortalityRateThreshold) {
+ // This is the oldest cohort that has high mortality.
+ // We ignore any cohorts that had a very low population count, or
+ // that have a lower mortality rate than we care to age in young; these
+ // cohorts are considered eligible for tenuring when all older
+ // cohorts are. We return the next higher age as the tenuring threshold
+ // so that we do not prematurely promote objects of this age.
+ assert(tenuring_threshold == i+1 || tenuring_threshold == upper_bound, "Error");
+ assert(tenuring_threshold >= lower_bound && tenuring_threshold <= upper_bound, "Error");
+ return tenuring_threshold;
+ }
+ // Remember that we passed over this cohort, looking for younger cohorts
+ // showing high mortality. We want to tenure cohorts of this age.
+ tenuring_threshold = i;
+ }
+ assert(tenuring_threshold >= lower_bound && tenuring_threshold <= upper_bound, "Error");
+ return tenuring_threshold;
+}
+
+// Mortality rate of a cohort, given its previous and current population
+double ShenandoahAgeCensus::mortality_rate(size_t prev_pop, size_t cur_pop) {
+ // The following also covers the case where both entries are 0
+ if (prev_pop <= cur_pop) {
+ // adjust for inaccurate censuses by finessing the
+ // reappearance of dark matter as normal matter;
+ // mortality rate is 0 if population remained the same
+ // or increased.
+ if (cur_pop > prev_pop) {
+ log_trace(gc, age)
+ (" (dark matter) Cohort population " SIZE_FORMAT_W(10) " to " SIZE_FORMAT_W(10),
+ prev_pop*oopSize, cur_pop*oopSize);
+ }
+ return 0.0;
+ }
+ assert(prev_pop > 0 && prev_pop > cur_pop, "Error");
+ return 1.0 - (((double)cur_pop)/((double)prev_pop));
+}
+
+void ShenandoahAgeCensus::print() {
+ // Print the population vector for the current epoch, and
+ // for the previous epoch, as well as the computed mortality
+ // ratio for each extant cohort.
+ const uint cur_epoch = _epoch;
+ const uint prev_epoch = cur_epoch > 0 ? cur_epoch - 1: markWord::max_age;
+
+ const AgeTable* cur_pv = _global_age_table[cur_epoch];
+ const AgeTable* prev_pv = _global_age_table[prev_epoch];
+
+ const uint tt = tenuring_threshold();
+
+ size_t total= 0;
+ for (uint i = 1; i < MAX_COHORTS; i++) {
+ const size_t prev_pop = prev_pv->sizes[i-1]; // (i-1) OK because i >= 1
+ const size_t cur_pop = cur_pv->sizes[i];
+ double mr = mortality_rate(prev_pop, cur_pop);
+ // Suppress printing when everything is zero
+ if (prev_pop + cur_pop > 0) {
+ log_info(gc, age)
+ (" - age %3u: prev " SIZE_FORMAT_W(10) " bytes, curr " SIZE_FORMAT_W(10) " bytes, mortality %.2f ",
+ i, prev_pop*oopSize, cur_pop*oopSize, mr);
+ }
+ total += cur_pop;
+ if (i == tt) {
+ // Underline the cohort for tenuring threshold (if < MAX_COHORTS)
+ log_info(gc, age)("----------------------------------------------------------------------------");
+ }
+ }
+ CENSUS_NOISE(_global_noise[cur_epoch].print(total);)
+}
+
+#ifdef SHENANDOAH_CENSUS_NOISE
+void ShenandoahNoiseStats::print(size_t total) {
+ if (total > 0) {
+ float f_skipped = (float)skipped/(float)total;
+ float f_aged = (float)aged/(float)total;
+ float f_clamped = (float)clamped/(float)total;
+ float f_young = (float)young/(float)total;
+ log_info(gc, age)("Skipped: " SIZE_FORMAT_W(10) " (%.2f), R-Aged: " SIZE_FORMAT_W(10) " (%.2f), "
+ "Clamped: " SIZE_FORMAT_W(10) " (%.2f), R-Young: " SIZE_FORMAT_W(10) " (%.2f)",
+ skipped*oopSize, f_skipped, aged*oopSize, f_aged,
+ clamped*oopSize, f_clamped, young*oopSize, f_young);
+ }
+}
+#endif // SHENANDOAH_CENSUS_NOISE
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp
new file mode 100644
index 00000000000..89c68f7120b
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp
@@ -0,0 +1,229 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHAGECENSUS_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHAGECENSUS_HPP
+
+#include "gc/shared/ageTable.hpp"
+
+#ifndef PRODUCT
+// Enable noise instrumentation
+#define SHENANDOAH_CENSUS_NOISE 1
+#endif // PRODUCT
+
+#ifdef SHENANDOAH_CENSUS_NOISE
+
+#define CENSUS_NOISE(x) x
+#define NO_CENSUS_NOISE(x)
+
+struct ShenandoahNoiseStats {
+ size_t skipped; // Volume of objects skipped
+ size_t aged; // Volume of objects from aged regions
+ size_t clamped; // Volume of objects whose ages were clamped
+ size_t young; // Volume of (rejuvenated) objects of retrograde age
+
+ ShenandoahNoiseStats() {
+ clear();
+ }
+
+ void clear() {
+ skipped = 0;
+ aged = 0;
+ clamped = 0;
+ young = 0;
+ }
+
+#ifndef PRODUCT
+ bool is_clear() {
+ return (skipped + aged + clamped + young) == 0;
+ }
+#endif // !PRODUCT
+
+ void merge(ShenandoahNoiseStats& other) {
+ skipped += other.skipped;
+ aged += other.aged;
+ clamped += other.clamped;
+ young += other.young;
+ }
+
+ void print(size_t total);
+};
+#else // SHENANDOAH_CENSUS_NOISE
+#define CENSUS_NOISE(x)
+#define NO_CENSUS_NOISE(x) x
+#endif // SHENANDOAH_CENSUS_NOISE
+
+// A class for tracking a sequence of cohort population vectors (or,
+// interchangeably, age tables) for up to C=MAX_COHORTS age cohorts, where a cohort
+// represents the set of objects allocated during a specific inter-GC epoch.
+// Epochs are demarcated by GC cycles, with those surviving a cycle aging by
+// an epoch. The census tracks the historical variation of cohort demographics
+// across N=MAX_SNAPSHOTS recent epochs. Since there are at most C age cohorts in
+// the population, we need only track at most N=C epochal snapshots to track a
+// maximal longitudinal demographics of every object's longitudinal cohort in
+// the young generation. The _global_age_table is thus, currently, a C x N (row-major)
+// matrix, with C=16, and, for now N=C=16, currently.
+// In theory, we might decide to track even longer (N=MAX_SNAPSHOTS) demographic
+// histories, but that isn't the case today. In particular, the current tenuring
+// threshold algorithm uses only 2 most recent snapshots, with the remaining
+// MAX_SNAPSHOTS-2=14 reserved for research purposes.
+//
+// In addition, this class also maintains per worker population vectors into which
+// census for the current minor GC is accumulated (during marking or, optionally, during
+// evacuation). These are cleared after each marking (resectively, evacuation) cycle,
+// once the per-worker data is consolidated into the appropriate population vector
+// per minor collection. The _local_age_table is thus C x N, for N GC workers.
+class ShenandoahAgeCensus: public CHeapObj {
+ AgeTable** _global_age_table; // Global age table used for adapting tenuring threshold, one per snapshot
+ AgeTable** _local_age_table; // Local scratch age tables to track object ages, one per worker
+
+#ifdef SHENANDOAH_CENSUS_NOISE
+ ShenandoahNoiseStats* _global_noise; // Noise stats, one per snapshot
+ ShenandoahNoiseStats* _local_noise; // Local scratch table for noise stats, one per worker
+
+ size_t _skipped; // net size of objects encountered, but skipped during census,
+ // because their age was indeterminate
+#endif // SHENANDOAH_CENSUS_NOISE
+
+#ifndef PRODUCT
+ size_t _counted; // net size of objects counted in census
+ size_t _total; // net size of objects encountered (counted or skipped) in census
+#endif
+
+ uint _epoch; // Current epoch (modulo max age)
+ uint *_tenuring_threshold; // An array of the last N tenuring threshold values we
+ // computed.
+
+ // Mortality rate of a cohort, given its population in
+ // previous and current epochs
+ double mortality_rate(size_t prev_pop, size_t cur_pop);
+
+ // Update to a new epoch, creating a slot for new census.
+ void prepare_for_census_update();
+
+ // Update the tenuring threshold, calling
+ // compute_tenuring_threshold() to calculate the new
+ // value
+ void update_tenuring_threshold();
+
+ // Use _global_age_table and the current _epoch to compute a new tenuring
+ // threshold, which will be remembered until the next invocation of
+ // compute_tenuring_threshold.
+ uint compute_tenuring_threshold();
+
+ // Return the tenuring threshold computed for the previous epoch
+ uint previous_tenuring_threshold() const {
+ assert(_epoch < MAX_SNAPSHOTS, "Error");
+ uint prev = _epoch - 1;
+ if (prev >= MAX_SNAPSHOTS) {
+ // _epoch is 0
+ assert(_epoch == 0, "Error");
+ prev = MAX_SNAPSHOTS - 1;
+ }
+ return _tenuring_threshold[prev];
+ }
+
+#ifndef PRODUCT
+ // Return the sum of size of objects of all ages recorded in the
+ // census at snapshot indexed by snap.
+ size_t get_all_ages(uint snap);
+
+ // Return the size of all objects that were encountered, but skipped,
+ // during the census, because their age was indeterminate.
+ size_t get_skipped(uint snap);
+
+ // Update the total size of objects counted or skipped at the census for
+ // the most recent epoch.
+ void update_total();
+#endif // !PRODUCT
+
+ public:
+ enum {
+ MAX_COHORTS = AgeTable::table_size, // = markWord::max_age + 1
+ MAX_SNAPSHOTS = MAX_COHORTS // May change in the future
+ };
+
+ ShenandoahAgeCensus();
+
+ // Return the local age table (population vector) for worker_id.
+ // Only used in the case of (ShenandoahGenerationalAdaptiveTenuring && !ShenandoahGenerationalCensusAtEvac)
+ AgeTable* get_local_age_table(uint worker_id) {
+ return (AgeTable*) _local_age_table[worker_id];
+ }
+
+ // Update the local age table for worker_id by size for
+ // given obj_age, region_age, and region_youth
+ CENSUS_NOISE(void add(uint obj_age, uint region_age, uint region_youth, size_t size, uint worker_id);)
+ NO_CENSUS_NOISE(void add(uint obj_age, uint region_age, size_t size, uint worker_id);)
+
+#ifdef SHENANDOAH_CENSUS_NOISE
+ // Update the local skip table for worker_id by size
+ void add_skipped(size_t size, uint worker_id);
+ // Update the local aged region volume table for worker_id by size
+ void add_aged(size_t size, uint worker_id);
+ // Update the local clamped object volume table for worker_id by size
+ void add_clamped(size_t size, uint worker_id);
+ // Update the local (rejuvenated) object volume (retrograde age) for worker_id by size
+ void add_young(size_t size, uint worker_id);
+#endif // SHENANDOAH_CENSUS_NOISE
+
+ // Update the census data, and compute the new tenuring threshold.
+ // This method should be called at the end of each marking (or optionally
+ // evacuation) cycle to update the tenuring threshold to be used in
+ // the next cycle.
+ // age0_pop is the population of Cohort 0 that may have been missed in
+ // the regular census during the marking cycle, corresponding to objects
+ // allocated when the concurrent marking was in progress.
+ // Optional parameters, pv1 and pv2 are population vectors that together
+ // provide object census data (only) for the case when
+ // ShenandoahGenerationalCensusAtEvac. In this case, the age0_pop
+ // is 0, because the evacuated objects have all had their ages incremented.
+ void update_census(size_t age0_pop, AgeTable* pv1 = nullptr, AgeTable* pv2 = nullptr);
+
+ // Return the most recently computed tenuring threshold
+ uint tenuring_threshold() const { return _tenuring_threshold[_epoch]; }
+
+ // Reset the epoch, clearing accumulated census history
+ // Note: this isn't currently used, but reserved for planned
+ // future usage.
+ void reset_global();
+
+ // Reset any (potentially partial) census information in worker-local age tables
+ void reset_local();
+
+#ifndef PRODUCT
+ // Check whether census information is clear
+ bool is_clear_global();
+ bool is_clear_local();
+
+ // Return the net size of objects encountered (counted or skipped) in census
+ // at most recent epoch.
+ size_t get_total() { return _total; }
+#endif // !PRODUCT
+
+ // Print the age census information
+ void print();
+};
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHAGECENSUS_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp
index 59b4615d326..6ac62e8e9ed 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,15 +26,17 @@
#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHALLOCREQUEST_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHALLOCREQUEST_HPP
+#include "gc/shenandoah/shenandoahAffiliation.hpp"
#include "memory/allocation.hpp"
class ShenandoahAllocRequest : StackObj {
public:
enum Type {
_alloc_shared, // Allocate common, outside of TLAB
- _alloc_shared_gc, // Allocate common, outside of GCLAB
+ _alloc_shared_gc, // Allocate common, outside of GCLAB/PLAB
_alloc_tlab, // Allocate TLAB
_alloc_gclab, // Allocate GCLAB
+ _alloc_plab, // Allocate PLAB
_ALLOC_LIMIT
};
@@ -47,6 +50,8 @@ class ShenandoahAllocRequest : StackObj {
return "TLAB";
case _alloc_gclab:
return "GCLAB";
+ case _alloc_plab:
+ return "PLAB";
default:
ShouldNotReachHere();
return "";
@@ -54,17 +59,38 @@ class ShenandoahAllocRequest : StackObj {
}
private:
+ // When ShenandoahElasticTLAB is enabled, the request cannot be made smaller than _min_size.
size_t _min_size;
+
+ // The size of the request in words.
size_t _requested_size;
+
+ // The allocation may be increased for padding or decreased to fit in the remaining space of a region.
size_t _actual_size;
+
+ // For a humongous object, the _waste is the amount of free memory in the last region.
+ // For other requests, the _waste will be non-zero if the request enountered one or more regions
+ // with less memory than _min_size. This waste does not contribute to the used memory for
+ // the heap, but it does contribute to the allocation rate for heuristics.
+ size_t _waste;
+
+ // This is the type of the request.
Type _alloc_type;
+
+ // This is the generation which the request is targeting.
+ ShenandoahAffiliation const _affiliation;
+
+ // True if this request is trying to copy any object from young to old (promote).
+ bool _is_promotion;
+
#ifdef ASSERT
+ // Check that this is set before being read.
bool _actual_size_set;
#endif
- ShenandoahAllocRequest(size_t _min_size, size_t _requested_size, Type _alloc_type) :
+ ShenandoahAllocRequest(size_t _min_size, size_t _requested_size, Type _alloc_type, ShenandoahAffiliation affiliation, bool is_promotion = false) :
_min_size(_min_size), _requested_size(_requested_size),
- _actual_size(0), _alloc_type(_alloc_type)
+ _actual_size(0), _waste(0), _alloc_type(_alloc_type), _affiliation(affiliation), _is_promotion(is_promotion)
#ifdef ASSERT
, _actual_size_set(false)
#endif
@@ -72,39 +98,47 @@ class ShenandoahAllocRequest : StackObj {
public:
static inline ShenandoahAllocRequest for_tlab(size_t min_size, size_t requested_size) {
- return ShenandoahAllocRequest(min_size, requested_size, _alloc_tlab);
+ return ShenandoahAllocRequest(min_size, requested_size, _alloc_tlab, ShenandoahAffiliation::YOUNG_GENERATION);
}
static inline ShenandoahAllocRequest for_gclab(size_t min_size, size_t requested_size) {
- return ShenandoahAllocRequest(min_size, requested_size, _alloc_gclab);
+ return ShenandoahAllocRequest(min_size, requested_size, _alloc_gclab, ShenandoahAffiliation::YOUNG_GENERATION);
+ }
+
+ static inline ShenandoahAllocRequest for_plab(size_t min_size, size_t requested_size) {
+ return ShenandoahAllocRequest(min_size, requested_size, _alloc_plab, ShenandoahAffiliation::OLD_GENERATION);
}
- static inline ShenandoahAllocRequest for_shared_gc(size_t requested_size) {
- return ShenandoahAllocRequest(0, requested_size, _alloc_shared_gc);
+ static inline ShenandoahAllocRequest for_shared_gc(size_t requested_size, ShenandoahAffiliation affiliation, bool is_promotion = false) {
+ if (is_promotion) {
+ assert(affiliation == ShenandoahAffiliation::OLD_GENERATION, "Should only promote to old generation");
+ return ShenandoahAllocRequest(0, requested_size, _alloc_shared_gc, affiliation, true);
+ }
+ return ShenandoahAllocRequest(0, requested_size, _alloc_shared_gc, affiliation);
}
static inline ShenandoahAllocRequest for_shared(size_t requested_size) {
- return ShenandoahAllocRequest(0, requested_size, _alloc_shared);
+ return ShenandoahAllocRequest(0, requested_size, _alloc_shared, ShenandoahAffiliation::YOUNG_GENERATION);
}
- inline size_t size() {
+ inline size_t size() const {
return _requested_size;
}
- inline Type type() {
+ inline Type type() const {
return _alloc_type;
}
- inline const char* type_string() {
+ inline const char* type_string() const {
return alloc_type_to_string(_alloc_type);
}
- inline size_t min_size() {
+ inline size_t min_size() const {
assert (is_lab_alloc(), "Only access for LAB allocs");
return _min_size;
}
- inline size_t actual_size() {
+ inline size_t actual_size() const {
assert (_actual_size_set, "Should be set");
return _actual_size;
}
@@ -117,12 +151,21 @@ class ShenandoahAllocRequest : StackObj {
_actual_size = v;
}
- inline bool is_mutator_alloc() {
+ inline size_t waste() const {
+ return _waste;
+ }
+
+ inline void set_waste(size_t v) {
+ _waste = v;
+ }
+
+ inline bool is_mutator_alloc() const {
switch (_alloc_type) {
case _alloc_tlab:
case _alloc_shared:
return true;
case _alloc_gclab:
+ case _alloc_plab:
case _alloc_shared_gc:
return false;
default:
@@ -131,12 +174,13 @@ class ShenandoahAllocRequest : StackObj {
}
}
- inline bool is_gc_alloc() {
+ inline bool is_gc_alloc() const {
switch (_alloc_type) {
case _alloc_tlab:
case _alloc_shared:
return false;
case _alloc_gclab:
+ case _alloc_plab:
case _alloc_shared_gc:
return true;
default:
@@ -145,10 +189,11 @@ class ShenandoahAllocRequest : StackObj {
}
}
- inline bool is_lab_alloc() {
+ inline bool is_lab_alloc() const {
switch (_alloc_type) {
case _alloc_tlab:
case _alloc_gclab:
+ case _alloc_plab:
return true;
case _alloc_shared:
case _alloc_shared_gc:
@@ -158,6 +203,26 @@ class ShenandoahAllocRequest : StackObj {
return false;
}
}
+
+ bool is_old() const {
+ return _affiliation == OLD_GENERATION;
+ }
+
+ bool is_young() const {
+ return _affiliation == YOUNG_GENERATION;
+ }
+
+ ShenandoahAffiliation affiliation() const {
+ return _affiliation;
+ }
+
+ const char* affiliation_name() const {
+ return shenandoah_affiliation_name(_affiliation);
+ }
+
+ bool is_promotion() const {
+ return _is_promotion;
+ }
};
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHALLOCREQUEST_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp
index 4376e9d1150..fa3f9019af4 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -29,6 +30,7 @@
#include "gc/shared/workerPolicy.hpp"
#include "gc/shenandoah/shenandoahArguments.hpp"
#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "runtime/globals_extension.hpp"
@@ -50,6 +52,7 @@ void ShenandoahArguments::initialize() {
FLAG_SET_DEFAULT(ShenandoahSATBBarrier, false);
FLAG_SET_DEFAULT(ShenandoahLoadRefBarrier, false);
FLAG_SET_DEFAULT(ShenandoahCASBarrier, false);
+ FLAG_SET_DEFAULT(ShenandoahCardBarrier, false);
FLAG_SET_DEFAULT(ShenandoahCloneBarrier, false);
FLAG_SET_DEFAULT(ShenandoahVerifyOptoBarriers, false);
@@ -69,6 +72,13 @@ void ShenandoahArguments::initialize() {
FLAG_SET_DEFAULT(UseNUMA, true);
}
+ // We use this as the time period for tracking minimum mutator utilization (MMU).
+ // In generational mode, the MMU is used as a signal to adjust the size of the
+ // young generation.
+ if (FLAG_IS_DEFAULT(GCPauseIntervalMillis)) {
+ FLAG_SET_DEFAULT(GCPauseIntervalMillis, 5000);
+ }
+
// Set up default number of concurrent threads. We want to have cycles complete fast
// enough, but we also do not want to steal too much CPU from the concurrently running
// application. Using 1/4 of available threads for concurrent GC seems a good
@@ -189,6 +199,8 @@ size_t ShenandoahArguments::conservative_max_heap_alignment() {
}
void ShenandoahArguments::initialize_alignments() {
+ CardTable::initialize_card_size();
+
// Need to setup sizes early to get correct alignments.
MaxHeapSize = ShenandoahHeapRegion::setup_sizes(MaxHeapSize);
@@ -202,5 +214,10 @@ void ShenandoahArguments::initialize_alignments() {
}
CollectedHeap* ShenandoahArguments::create_heap() {
- return new ShenandoahHeap(new ShenandoahCollectorPolicy());
+ if (strcmp(ShenandoahGCMode, "generational") != 0) {
+ // Not generational
+ return new ShenandoahHeap(new ShenandoahCollectorPolicy());
+ } else {
+ return new ShenandoahGenerationalHeap(new ShenandoahCollectorPolicy());
+ }
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp
index 22822ad3fec..437646becce 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp
@@ -71,6 +71,9 @@ void ShenandoahAsserts::print_obj(ShenandoahMessageBuffer& msg, oop obj) {
msg.append(" %3s marked strong\n", ctx->is_marked_strong(obj) ? "" : "not");
msg.append(" %3s marked weak\n", ctx->is_marked_weak(obj) ? "" : "not");
msg.append(" %3s in collection set\n", heap->in_collection_set(obj) ? "" : "not");
+ if (heap->mode()->is_generational() && !obj->is_forwarded()) {
+ msg.append(" age: %d\n", obj->age());
+ }
msg.append(" mark:%s\n", mw_ss.freeze());
msg.append(" region: %s", ss.freeze());
}
@@ -424,7 +427,7 @@ void ShenandoahAsserts::assert_locked_or_shenandoah_safepoint(Mutex* lock, const
return;
}
- ShenandoahMessageBuffer msg("Must ba at a Shenandoah safepoint or held %s lock", lock->name());
+ ShenandoahMessageBuffer msg("Must be at a Shenandoah safepoint or held %s lock", lock->name());
report_vm_error(file, line, msg.buffer());
}
@@ -457,10 +460,55 @@ void ShenandoahAsserts::assert_heaplocked_or_safepoint(const char* file, int lin
return;
}
- if (ShenandoahSafepoint::is_at_shenandoah_safepoint() && Thread::current()->is_VM_thread()) {
+ if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
return;
}
ShenandoahMessageBuffer msg("Heap lock must be owned by current thread, or be at safepoint");
report_vm_error(file, line, msg.buffer());
}
+
+void ShenandoahAsserts::assert_generational(const char* file, int line) {
+ if (ShenandoahHeap::heap()->mode()->is_generational()) {
+ return;
+ }
+
+ ShenandoahMessageBuffer msg("Must be in generational mode");
+ report_vm_error(file, line, msg.buffer());
+}
+
+void ShenandoahAsserts::assert_control_or_vm_thread_at_safepoint(bool at_safepoint, const char* file, int line) {
+ Thread* thr = Thread::current();
+ if (thr == ShenandoahHeap::heap()->control_thread()) {
+ return;
+ }
+ if (thr->is_VM_thread()) {
+ if (!at_safepoint) {
+ return;
+ } else if (SafepointSynchronize::is_at_safepoint()) {
+ return;
+ }
+ }
+
+ ShenandoahMessageBuffer msg("Must be either control thread, or vm thread");
+ if (at_safepoint) {
+ msg.append(" at a safepoint");
+ }
+ report_vm_error(file, line, msg.buffer());
+}
+
+void ShenandoahAsserts::assert_generations_reconciled(const char* file, int line) {
+ if (!SafepointSynchronize::is_at_safepoint()) {
+ return;
+ }
+
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+ ShenandoahGeneration* ggen = heap->gc_generation();
+ ShenandoahGeneration* agen = heap->active_generation();
+ if (agen == ggen) {
+ return;
+ }
+
+ ShenandoahMessageBuffer msg("Active(%d) & GC(%d) Generations aren't reconciled", agen->type(), ggen->type());
+ report_vm_error(file, line, msg.buffer());
+}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp
index 154edebcf3e..31a99bf438c 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -72,6 +73,9 @@ class ShenandoahAsserts {
static void assert_heaplocked(const char* file, int line);
static void assert_not_heaplocked(const char* file, int line);
static void assert_heaplocked_or_safepoint(const char* file, int line);
+ static void assert_control_or_vm_thread_at_safepoint(bool at_safepoint, const char* file, int line);
+ static void assert_generational(const char* file, int line);
+ static void assert_generations_reconciled(const char* file, int line);
#ifdef ASSERT
#define shenandoah_assert_in_heap_bounds(interior_loc, obj) \
@@ -163,6 +167,21 @@ class ShenandoahAsserts {
#define shenandoah_assert_heaplocked_or_safepoint() \
ShenandoahAsserts::assert_heaplocked_or_safepoint(__FILE__, __LINE__)
+
+#define shenandoah_assert_control_or_vm_thread() \
+ ShenandoahAsserts::assert_control_or_vm_thread(false /* at_safepoint */, __FILE__, __LINE__)
+
+// A stronger version of the above that checks that we are at a safepoint if the vm thread
+#define shenandoah_assert_control_or_vm_thread_at_safepoint() \
+ ShenandoahAsserts::assert_control_or_vm_thread_at_safepoint(true /* at_safepoint */, __FILE__, __LINE__)
+
+#define shenandoah_assert_generational() \
+ ShenandoahAsserts::assert_generational(__FILE__, __LINE__)
+
+// Some limited sanity checking of the _gc_generation and _active_generation fields of ShenandoahHeap
+#define shenandoah_assert_generations_reconciled() \
+ ShenandoahAsserts::assert_generations_reconciled(__FILE__, __LINE__)
+
#else
#define shenandoah_assert_in_heap_bounds(interior_loc, obj)
#define shenandoah_assert_in_heap_bounds_or_null(interior_loc, obj)
@@ -213,6 +232,10 @@ class ShenandoahAsserts {
#define shenandoah_assert_heaplocked()
#define shenandoah_assert_not_heaplocked()
#define shenandoah_assert_heaplocked_or_safepoint()
+#define shenandoah_assert_control_or_vm_thread()
+#define shenandoah_assert_control_or_vm_thread_at_safepoint()
+#define shenandoah_assert_generational()
+#define shenandoah_assert_generations_reconciled()
#endif
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
index 40b6d70ce45..38e363664dc 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,8 +29,10 @@
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
#include "gc/shenandoah/shenandoahBarrierSetNMethod.hpp"
#include "gc/shenandoah/shenandoahBarrierSetStackChunk.hpp"
+#include "gc/shenandoah/shenandoahCardTable.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
#include "gc/shenandoah/shenandoahStackWatermark.hpp"
#ifdef COMPILER1
#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
@@ -41,7 +44,7 @@
class ShenandoahBarrierSetC1;
class ShenandoahBarrierSetC2;
-ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) :
+ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap, MemRegion heap_region) :
BarrierSet(make_barrier_set_assembler(),
make_barrier_set_c1(),
make_barrier_set_c2(),
@@ -49,9 +52,14 @@ ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) :
new ShenandoahBarrierSetStackChunk(),
BarrierSet::FakeRtti(BarrierSet::ShenandoahBarrierSet)),
_heap(heap),
+ _card_table(nullptr),
_satb_mark_queue_buffer_allocator("SATB Buffer Allocator", ShenandoahSATBBufferSize),
_satb_mark_queue_set(&_satb_mark_queue_buffer_allocator)
{
+ if (ShenandoahCardBarrier) {
+ _card_table = new ShenandoahCardTable(heap_region);
+ _card_table->initialize();
+ }
}
ShenandoahBarrierSetAssembler* ShenandoahBarrierSet::assembler() {
@@ -124,6 +132,12 @@ void ShenandoahBarrierSet::on_thread_detach(Thread *thread) {
gclab->retire();
}
+ PLAB* plab = ShenandoahThreadLocalData::plab(thread);
+ if (plab != nullptr) {
+ // This will assert if plab is not null in non-generational mode
+ ShenandoahGenerationalHeap::heap()->retire_plab(plab);
+ }
+
// SATB protocol requires to keep alive reachable oops from roots at the beginning of GC
if (ShenandoahStackWatermarkBarrier) {
if (_heap->is_concurrent_mark_in_progress()) {
@@ -142,3 +156,22 @@ void ShenandoahBarrierSet::clone_barrier_runtime(oop src) {
clone_barrier(src);
}
}
+
+void ShenandoahBarrierSet::write_ref_array(HeapWord* start, size_t count) {
+ assert(ShenandoahCardBarrier, "Should have been checked by caller");
+
+ HeapWord* end = (HeapWord*)((char*) start + (count * heapOopSize));
+ // In the case of compressed oops, start and end may potentially be misaligned;
+ // so we need to conservatively align the first downward (this is not
+ // strictly necessary for current uses, but a case of good hygiene and,
+ // if you will, aesthetics) and the second upward (this is essential for
+ // current uses) to a HeapWord boundary, so we mark all cards overlapping
+ // this write.
+ HeapWord* aligned_start = align_down(start, HeapWordSize);
+ HeapWord* aligned_end = align_up (end, HeapWordSize);
+ // If compressed oops were not being used, these should already be aligned
+ assert(UseCompressedOops || (aligned_start == start && aligned_end == end),
+ "Expected heap word alignment of start and end");
+ _heap->old_generation()->card_scan()->mark_range_as_dirty(aligned_start, (aligned_end - aligned_start));
+}
+
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
index e116e0225db..8d1dc92761a 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -31,15 +32,17 @@
class ShenandoahHeap;
class ShenandoahBarrierSetAssembler;
+class ShenandoahCardTable;
class ShenandoahBarrierSet: public BarrierSet {
private:
ShenandoahHeap* const _heap;
+ ShenandoahCardTable* _card_table;
BufferNode::Allocator _satb_mark_queue_buffer_allocator;
ShenandoahSATBMarkQueueSet _satb_mark_queue_set;
public:
- ShenandoahBarrierSet(ShenandoahHeap* heap);
+ ShenandoahBarrierSet(ShenandoahHeap* heap, MemRegion heap_region);
static ShenandoahBarrierSetAssembler* assembler();
@@ -47,6 +50,10 @@ class ShenandoahBarrierSet: public BarrierSet {
return barrier_set_cast(BarrierSet::barrier_set());
}
+ inline ShenandoahCardTable* card_table() {
+ return _card_table;
+ }
+
static ShenandoahSATBMarkQueueSet& satb_mark_queue_set() {
return barrier_set()->_satb_mark_queue_set;
}
@@ -111,9 +118,14 @@ class ShenandoahBarrierSet: public BarrierSet {
template
inline oop oop_xchg(DecoratorSet decorators, T* addr, oop new_value);
+ template
+ void write_ref_field_post(T* field);
+
+ void write_ref_array(HeapWord* start, size_t count);
+
private:
template
- inline void arraycopy_marking(T* src, T* dst, size_t count);
+ inline void arraycopy_marking(T* src, T* dst, size_t count, bool is_old_marking);
template
inline void arraycopy_evacuation(T* src, size_t count);
template
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
index 80735f453c6..f3ce9418d35 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2015, 2022, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,14 +29,19 @@
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
#include "gc/shared/accessBarrierSupport.inline.hpp"
+#include "gc/shared/cardTable.hpp"
#include "gc/shenandoah/shenandoahAsserts.hpp"
+#include "gc/shenandoah/shenandoahCardTable.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
#include "gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp"
#include "gc/shenandoah/shenandoahForwarding.inline.hpp"
+#include "gc/shenandoah/shenandoahGeneration.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "gc/shenandoah/mode/shenandoahMode.hpp"
+#include "memory/iterator.inline.hpp"
#include "oops/oop.inline.hpp"
inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) {
@@ -103,6 +109,7 @@ inline oop ShenandoahBarrierSet::load_reference_barrier(DecoratorSet decorators,
// Prevent resurrection of unreachable phantom (i.e. weak-native) references.
if ((decorators & ON_PHANTOM_OOP_REF) != 0 &&
_heap->is_concurrent_weak_root_in_progress() &&
+ _heap->is_in_active_generation(obj) &&
!_heap->marking_context()->is_marked(obj)) {
return nullptr;
}
@@ -110,6 +117,7 @@ inline oop ShenandoahBarrierSet::load_reference_barrier(DecoratorSet decorators,
// Prevent resurrection of unreachable weak references.
if ((decorators & ON_WEAK_OOP_REF) != 0 &&
_heap->is_concurrent_weak_root_in_progress() &&
+ _heap->is_in_active_generation(obj) &&
!_heap->marking_context()->is_marked_strong(obj)) {
return nullptr;
}
@@ -173,6 +181,13 @@ inline void ShenandoahBarrierSet::keep_alive_if_weak(DecoratorSet decorators, oo
}
}
+template
+inline void ShenandoahBarrierSet::write_ref_field_post(T* field) {
+ assert(ShenandoahCardBarrier, "Should have been checked by caller");
+ volatile CardTable::CardValue* byte = card_table()->byte_for(field);
+ *byte = CardTable::dirty_card_val();
+}
+
template
inline oop ShenandoahBarrierSet::oop_load(DecoratorSet decorators, T* addr) {
oop value = RawAccess<>::oop_load(addr);
@@ -234,7 +249,10 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_loa
template
template
inline void ShenandoahBarrierSet::AccessBarrier::oop_store_common(T* addr, oop value) {
- shenandoah_assert_marked_if(nullptr, value, !CompressedOops::is_null(value) && ShenandoahHeap::heap()->is_evacuation_in_progress());
+ shenandoah_assert_marked_if(nullptr, value,
+ !CompressedOops::is_null(value) && ShenandoahHeap::heap()->is_evacuation_in_progress()
+ && !(ShenandoahHeap::heap()->active_generation()->is_young()
+ && ShenandoahHeap::heap()->heap_region_containing(value)->is_old()));
shenandoah_assert_not_in_cset_if(addr, value, value != nullptr && !ShenandoahHeap::heap()->cancelled_gc());
ShenandoahBarrierSet* const bs = ShenandoahBarrierSet::barrier_set();
bs->satb_barrier(addr);
@@ -254,6 +272,10 @@ inline void ShenandoahBarrierSet::AccessBarrier::oop_st
shenandoah_assert_not_forwarded_except (addr, value, value == nullptr || ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahHeap::heap()->is_concurrent_mark_in_progress());
oop_store_common(addr, value);
+ if (ShenandoahCardBarrier) {
+ ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
+ bs->write_ref_field_post(addr);
+ }
}
template
@@ -274,7 +296,11 @@ template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap(T* addr, oop compare_value, oop new_value) {
assert((decorators & (AS_NO_KEEPALIVE | ON_UNKNOWN_OOP_REF)) == 0, "must be absent");
ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
- return bs->oop_cmpxchg(decorators, addr, compare_value, new_value);
+ oop result = bs->oop_cmpxchg(decorators, addr, compare_value, new_value);
+ if (ShenandoahCardBarrier) {
+ bs->write_ref_field_post(addr);
+ }
+ return result;
}
template
@@ -282,7 +308,12 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_ato
assert((decorators & AS_NO_KEEPALIVE) == 0, "must be absent");
ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
DecoratorSet resolved_decorators = AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset);
- return bs->oop_cmpxchg(resolved_decorators, AccessInternal::oop_field_addr(base, offset), compare_value, new_value);
+ auto addr = AccessInternal::oop_field_addr(base, offset);
+ oop result = bs->oop_cmpxchg(resolved_decorators, addr, compare_value, new_value);
+ if (ShenandoahCardBarrier) {
+ bs->write_ref_field_post(addr);
+ }
+ return result;
}
template
@@ -298,7 +329,11 @@ template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_xchg_in_heap(T* addr, oop new_value) {
assert((decorators & (AS_NO_KEEPALIVE | ON_UNKNOWN_OOP_REF)) == 0, "must be absent");
ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
- return bs->oop_xchg(decorators, addr, new_value);
+ oop result = bs->oop_xchg(decorators, addr, new_value);
+ if (ShenandoahCardBarrier) {
+ bs->write_ref_field_post(addr);
+ }
+ return result;
}
template
@@ -306,7 +341,12 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_ato
assert((decorators & AS_NO_KEEPALIVE) == 0, "must be absent");
ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
DecoratorSet resolved_decorators = AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset);
- return bs->oop_xchg(resolved_decorators, AccessInternal::oop_field_addr(base, offset), new_value);
+ auto addr = AccessInternal::oop_field_addr(base, offset);
+ oop result = bs->oop_xchg(resolved_decorators, addr, new_value);
+ if (ShenandoahCardBarrier) {
+ bs->write_ref_field_post(addr);
+ }
+ return result;
}
// Clone barrier support
@@ -323,16 +363,29 @@ template
bool ShenandoahBarrierSet::AccessBarrier::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
size_t length) {
+ T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
+ T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
+
ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
- bs->arraycopy_barrier(arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw),
- arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw),
- length);
- return Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
+ bs->arraycopy_barrier(src, dst, length);
+ bool result = Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
+ if (ShenandoahCardBarrier) {
+ bs->write_ref_array((HeapWord*) dst, length);
+ }
+ return result;
}
template
void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) {
- assert(HAS_FWD == _heap->has_forwarded_objects(), "Forwarded object status is sane");
+ // Young cycles are allowed to run when old marking is in progress. When old marking is in progress,
+ // this barrier will be called with ENQUEUE=true and HAS_FWD=false, even though the young generation
+ // may have forwarded objects. In this case, the `arraycopy_work` is first called with HAS_FWD=true and
+ // ENQUEUE=false.
+ assert(HAS_FWD == _heap->has_forwarded_objects() || (_heap->gc_state() & ShenandoahHeap::OLD_MARKING) != 0,
+ "Forwarded object status is sane");
+ // This function cannot be called to handle marking and evacuation at the same time (they operate on
+ // different sides of the copy).
+ assert((HAS_FWD || EVAC) != ENQUEUE, "Cannot evacuate and mark both sides of copy.");
Thread* thread = Thread::current();
SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread);
@@ -350,9 +403,8 @@ void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) {
}
shenandoah_assert_forwarded_except(elem_ptr, obj, _heap->cancelled_gc());
ShenandoahHeap::atomic_update_oop(fwd, elem_ptr, o);
- obj = fwd;
}
- if (ENQUEUE && !ctx->is_marked_strong(obj)) {
+ if (ENQUEUE && !ctx->is_marked_strong_or_old(obj)) {
_satb_mark_queue_set.enqueue_known_active(queue, obj);
}
}
@@ -362,24 +414,73 @@ void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) {
template
void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) {
if (count == 0) {
+ // No elements to copy, no need for barrier
return;
}
+
int gc_state = _heap->gc_state();
- if ((gc_state & ShenandoahHeap::MARKING) != 0) {
- arraycopy_marking(src, dst, count);
- } else if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
+ if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
arraycopy_evacuation(src, count);
} else if ((gc_state & ShenandoahHeap::UPDATEREFS) != 0) {
arraycopy_update(src, count);
}
+
+ if (_heap->mode()->is_generational()) {
+ assert(ShenandoahSATBBarrier, "Generational mode assumes SATB mode");
+ if ((gc_state & ShenandoahHeap::YOUNG_MARKING) != 0) {
+ arraycopy_marking(src, dst, count, false);
+ }
+ if ((gc_state & ShenandoahHeap::OLD_MARKING) != 0) {
+ arraycopy_marking(src, dst, count, true);
+ }
+ } else if ((gc_state & ShenandoahHeap::MARKING) != 0) {
+ arraycopy_marking(src, dst, count, false);
+ }
}
template
-void ShenandoahBarrierSet::arraycopy_marking(T* src, T* dst, size_t count) {
+void ShenandoahBarrierSet::arraycopy_marking(T* src, T* dst, size_t count, bool is_old_marking) {
assert(_heap->is_concurrent_mark_in_progress(), "only during marking");
- T* array = ShenandoahSATBBarrier ? dst : src;
- if (!_heap->marking_context()->allocated_after_mark_start(reinterpret_cast(array))) {
- arraycopy_work(array, count);
+ /*
+ * Note that an old-gen object is considered live if it is live at the start of OLD marking or if it is promoted
+ * following the start of OLD marking.
+ *
+ * 1. Every object promoted following the start of OLD marking will be above TAMS within its old-gen region
+ * 2. Every object live at the start of OLD marking will be referenced from a "root" or it will be referenced from
+ * another live OLD-gen object. With regards to old-gen, roots include stack locations and all of live young-gen.
+ * All root references to old-gen are identified during a bootstrap young collection. All references from other
+ * old-gen objects will be marked during the traversal of all old objects, or will be marked by the SATB barrier.
+ *
+ * During old-gen marking (which is interleaved with young-gen collections), call arraycopy_work() if:
+ *
+ * 1. The overwritten array resides in old-gen and it is below TAMS within its old-gen region
+ * 2. Do not call arraycopy_work for any array residing in young-gen because young-gen collection is idle at this time
+ *
+ * During young-gen marking, call arraycopy_work() if:
+ *
+ * 1. The overwritten array resides in young-gen and is below TAMS within its young-gen region
+ * 2. Additionally, if array resides in old-gen, regardless of its relationship to TAMS because this old-gen array
+ * may hold references to young-gen
+ */
+ if (ShenandoahSATBBarrier) {
+ T* array = dst;
+ HeapWord* array_addr = reinterpret_cast(array);
+ ShenandoahHeapRegion* r = _heap->heap_region_containing(array_addr);
+ if (is_old_marking) {
+ // Generational, old marking
+ assert(_heap->mode()->is_generational(), "Invariant");
+ if (r->is_old() && (array_addr < _heap->marking_context()->top_at_mark_start(r))) {
+ arraycopy_work(array, count);
+ }
+ } else if (_heap->mode()->is_generational()) {
+ // Generational, young marking
+ if (r->is_old() || (array_addr < _heap->marking_context()->top_at_mark_start(r))) {
+ arraycopy_work(array, count);
+ }
+ } else if (array_addr < _heap->marking_context()->top_at_mark_start(r)) {
+ // Non-generational, marking
+ arraycopy_work(array, count);
+ }
}
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp
new file mode 100644
index 00000000000..ef2d6e134b2
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp
@@ -0,0 +1,43 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+
+#include "precompiled.hpp"
+
+#include "gc/shenandoah/shenandoahCardStats.hpp"
+#include "logging/log.hpp"
+
+#ifndef PRODUCT
+void ShenandoahCardStats::log() const {
+ if (ShenandoahEnableCardStats) {
+ log_info(gc,remset)("Card stats: dirty " SIZE_FORMAT " (max run: " SIZE_FORMAT "),"
+ " clean " SIZE_FORMAT " (max run: " SIZE_FORMAT "),"
+ " dirty scans/objs " SIZE_FORMAT,
+ _dirty_card_cnt, _max_dirty_run,
+ _clean_card_cnt, _max_clean_run,
+ _dirty_scan_obj_cnt);
+ }
+}
+#endif // !PRODUCT
+
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp
new file mode 100644
index 00000000000..0ef0eaadb6b
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp
@@ -0,0 +1,132 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHCARDSTATS_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHCARDSTATS_HPP
+
+#include "gc/shared/gc_globals.hpp"
+#include "gc/shenandoah/shenandoahNumberSeq.hpp"
+
+enum CardStatType {
+ DIRTY_RUN,
+ CLEAN_RUN,
+ DIRTY_CARDS,
+ CLEAN_CARDS,
+ MAX_DIRTY_RUN,
+ MAX_CLEAN_RUN,
+ DIRTY_SCAN_OBJS,
+ ALTERNATIONS,
+ MAX_CARD_STAT_TYPE
+};
+
+enum CardStatLogType {
+ CARD_STAT_SCAN_RS,
+ CARD_STAT_UPDATE_REFS,
+ MAX_CARD_STAT_LOG_TYPE
+};
+
+class ShenandoahCardStats: public CHeapObj {
+private:
+ size_t _cards_in_cluster;
+ HdrSeq* _local_card_stats;
+
+ size_t _dirty_card_cnt;
+ size_t _clean_card_cnt;
+
+ size_t _max_dirty_run;
+ size_t _max_clean_run;
+
+ size_t _dirty_scan_obj_cnt;
+
+ size_t _alternation_cnt;
+
+public:
+ ShenandoahCardStats(size_t cards_in_cluster, HdrSeq* card_stats) :
+ _cards_in_cluster(cards_in_cluster),
+ _local_card_stats(card_stats),
+ _dirty_card_cnt(0),
+ _clean_card_cnt(0),
+ _max_dirty_run(0),
+ _max_clean_run(0),
+ _dirty_scan_obj_cnt(0),
+ _alternation_cnt(0)
+ { }
+
+ ~ShenandoahCardStats() {
+ record();
+ }
+
+ void record() {
+ if (ShenandoahEnableCardStats) {
+ // Update global stats for distribution of dirty/clean cards as a percentage of chunk
+ _local_card_stats[DIRTY_CARDS].add(percent_of(_dirty_card_cnt, _cards_in_cluster));
+ _local_card_stats[CLEAN_CARDS].add(percent_of(_clean_card_cnt, _cards_in_cluster));
+
+ // Update global stats for max dirty/clean run distribution as a percentage of chunk
+ _local_card_stats[MAX_DIRTY_RUN].add(percent_of(_max_dirty_run, _cards_in_cluster));
+ _local_card_stats[MAX_CLEAN_RUN].add(percent_of(_max_clean_run, _cards_in_cluster));
+
+ // Update global stats for dirty obj scan counts
+ _local_card_stats[DIRTY_SCAN_OBJS].add(_dirty_scan_obj_cnt);
+
+ // Update global stats for alternation counts
+ _local_card_stats[ALTERNATIONS].add(_alternation_cnt);
+ }
+ }
+
+public:
+ inline void record_dirty_run(size_t len) {
+ if (ShenandoahEnableCardStats) {
+ _alternation_cnt++;
+ if (len > _max_dirty_run) {
+ _max_dirty_run = len;
+ }
+ _dirty_card_cnt += len;
+ assert(len <= _cards_in_cluster, "Error");
+ _local_card_stats[DIRTY_RUN].add(percent_of(len, _cards_in_cluster));
+ }
+ }
+
+ inline void record_clean_run(size_t len) {
+ if (ShenandoahEnableCardStats) {
+ _alternation_cnt++;
+ if (len > _max_clean_run) {
+ _max_clean_run = len;
+ }
+ _clean_card_cnt += len;
+ assert(len <= _cards_in_cluster, "Error");
+ _local_card_stats[CLEAN_RUN].add(percent_of(len, _cards_in_cluster));
+ }
+ }
+
+ inline void record_scan_obj_cnt(size_t i) {
+ if (ShenandoahEnableCardStats) {
+ _dirty_scan_obj_cnt += i;
+ }
+ }
+
+ void log() const PRODUCT_RETURN;
+};
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHCARDSTATS_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp
new file mode 100644
index 00000000000..6ecb93717df
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp
@@ -0,0 +1,105 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shenandoah/shenandoahCardTable.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahUtils.hpp"
+#include "runtime/init.hpp"
+#include "nmt/memTracker.hpp"
+
+void ShenandoahCardTable::initialize() {
+ size_t num_cards = cards_required(_whole_heap.word_size());
+
+ // each card takes 1 byte; + 1 for the guard card
+ size_t num_bytes = num_cards + 1;
+ const size_t granularity = os::vm_allocation_granularity();
+ _byte_map_size = align_up(num_bytes, MAX2(_page_size, granularity));
+
+ HeapWord* low_bound = _whole_heap.start();
+ HeapWord* high_bound = _whole_heap.end();
+
+ // ReservedSpace constructor would assert rs_align >= os::vm_page_size().
+ const size_t rs_align = _page_size == os::vm_page_size() ? 0 : MAX2(_page_size, granularity);
+
+ ReservedSpace write_space(_byte_map_size, rs_align, _page_size);
+ initialize(write_space);
+
+ // The assembler store_check code will do an unsigned shift of the oop,
+ // then add it to _byte_map_base, i.e.
+ //
+ // _byte_map = _byte_map_base + (uintptr_t(low_bound) >> card_shift)
+ _byte_map = (CardValue*) write_space.base();
+ _byte_map_base = _byte_map - (uintptr_t(low_bound) >> _card_shift);
+ assert(byte_for(low_bound) == &_byte_map[0], "Checking start of map");
+ assert(byte_for(high_bound-1) <= &_byte_map[last_valid_index()], "Checking end of map");
+
+ _write_byte_map = _byte_map;
+ _write_byte_map_base = _byte_map_base;
+
+ ReservedSpace read_space(_byte_map_size, rs_align, _page_size);
+ initialize(read_space);
+
+ _read_byte_map = (CardValue*) read_space.base();
+ _read_byte_map_base = _read_byte_map - (uintptr_t(low_bound) >> card_shift());
+ assert(read_byte_for(low_bound) == &_read_byte_map[0], "Checking start of map");
+ assert(read_byte_for(high_bound-1) <= &_read_byte_map[last_valid_index()], "Checking end of map");
+
+ _covered[0] = _whole_heap;
+
+ log_trace(gc, barrier)("ShenandoahCardTable::ShenandoahCardTable:");
+ log_trace(gc, barrier)(" &_write_byte_map[0]: " INTPTR_FORMAT " &_write_byte_map[_last_valid_index]: " INTPTR_FORMAT,
+ p2i(&_write_byte_map[0]), p2i(&_write_byte_map[last_valid_index()]));
+ log_trace(gc, barrier)(" _write_byte_map_base: " INTPTR_FORMAT, p2i(_write_byte_map_base));
+ log_trace(gc, barrier)(" &_read_byte_map[0]: " INTPTR_FORMAT " &_read_byte_map[_last_valid_index]: " INTPTR_FORMAT,
+ p2i(&_read_byte_map[0]), p2i(&_read_byte_map[last_valid_index()]));
+ log_trace(gc, barrier)(" _read_byte_map_base: " INTPTR_FORMAT, p2i(_read_byte_map_base));
+}
+
+void ShenandoahCardTable::initialize(const ReservedSpace& card_table) {
+ MemTracker::record_virtual_memory_tag((address)card_table.base(), mtGC);
+
+ os::trace_page_sizes("Card Table", _byte_map_size, _byte_map_size,
+ card_table.base(), card_table.size(), _page_size);
+ if (!card_table.is_reserved()) {
+ vm_exit_during_initialization("Could not reserve enough space for the card marking array");
+ }
+ os::commit_memory_or_exit(card_table.base(), _byte_map_size, card_table.alignment(), false,
+ "Cannot commit memory for card table");
+}
+
+bool ShenandoahCardTable::is_in_young(const void* obj) const {
+ return ShenandoahHeap::heap()->is_in_young(obj);
+}
+
+CardValue* ShenandoahCardTable::read_byte_for(const void* p) {
+ CardValue* result = &_read_byte_map_base[uintptr_t(p) >> _card_shift];
+ assert(result >= _read_byte_map && result < _read_byte_map + _byte_map_size,
+ "out of bounds accessor for card marking array");
+ return result;
+}
+
+size_t ShenandoahCardTable::last_valid_index() {
+ return CardTable::last_valid_index();
+}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp
new file mode 100644
index 00000000000..f30ce09668a
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp
@@ -0,0 +1,87 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHCARDTABLE_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHCARDTABLE_HPP
+
+#include "gc/shared/cardTable.hpp"
+#include "memory/virtualspace.hpp"
+#include "oops/oopsHierarchy.hpp"
+#include "utilities/macros.hpp"
+
+class ShenandoahCardTable: public CardTable {
+ friend class VMStructs;
+
+private:
+ // We maintain two copies of the card table to facilitate concurrent remembered set scanning
+ // and concurrent clearing of stale remembered set information. During the init_mark safepoint,
+ // we copy the contents of _write_byte_map to _read_byte_map and clear _write_byte_map.
+ //
+ // Concurrent remembered set scanning reads from _read_byte_map while concurrent mutator write
+ // barriers are overwriting cards of the _write_byte_map with DIRTY codes. Concurrent remembered
+ // set scanning also overwrites cards of the _write_byte_map with DIRTY codes whenever it discovers
+ // interesting pointers.
+ //
+ // During a concurrent update-references phase, we scan the _write_byte_map concurrently to find
+ // all old-gen references that may need to be updated.
+ //
+ // In a future implementation, we may swap the values of _read_byte_map and _write_byte_map during
+ // the init-mark safepoint to avoid the need for bulk STW copying and initialization. Doing so
+ // requires a change to the implementation of mutator write barriers as the address of the card
+ // table is currently in-lined and hard-coded.
+ CardValue* _read_byte_map;
+ CardValue* _write_byte_map;
+ CardValue* _read_byte_map_base;
+ CardValue* _write_byte_map_base;
+
+public:
+ explicit ShenandoahCardTable(MemRegion whole_heap) : CardTable(whole_heap),
+ _read_byte_map(nullptr), _write_byte_map(nullptr),
+ _read_byte_map_base(nullptr), _write_byte_map_base(nullptr) {}
+
+ void initialize();
+
+ bool is_in_young(const void* obj) const override;
+
+ CardValue* read_byte_for(const void* p);
+
+ size_t last_valid_index();
+
+ CardValue* read_byte_map() {
+ return _read_byte_map;
+ }
+
+ CardValue* write_byte_map() {
+ return _write_byte_map;
+ }
+
+ CardValue* write_byte_map_base() {
+ return _write_byte_map_base;
+ }
+
+private:
+ void initialize(const ReservedSpace& card_table);
+};
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHCARDTABLE_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp
index 58527e808e4..fa7887145df 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp
@@ -58,6 +58,7 @@ class ShenandoahSuperClosure : public MetadataVisitingOopIterateClosure {
class ShenandoahMarkRefsSuperClosure : public ShenandoahSuperClosure {
private:
ShenandoahObjToScanQueue* _queue;
+ ShenandoahObjToScanQueue* _old_queue;
ShenandoahMarkingContext* const _mark_context;
bool _weak;
@@ -66,7 +67,7 @@ class ShenandoahMarkRefsSuperClosure : public ShenandoahSuperClosure {
void work(T *p);
public:
- inline ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp);
+ inline ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old_q);
bool is_weak() const {
return _weak;
@@ -89,8 +90,8 @@ class ShenandoahMarkRefsClosure : public ShenandoahMarkRefsSuperClosure {
inline void do_oop_work(T* p) { work(p); }
public:
- ShenandoahMarkRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
- ShenandoahMarkRefsSuperClosure(q, rp) {};
+ ShenandoahMarkRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old_q) :
+ ShenandoahMarkRefsSuperClosure(q, rp, old_q) {};
virtual void do_oop(narrowOop* p) { do_oop_work(p); }
virtual void do_oop(oop* p) { do_oop_work(p); }
@@ -191,7 +192,7 @@ class ShenandoahMarkUpdateRefsClosure : public ShenandoahMarkRefsSuperClosure {
inline void work(T* p);
public:
- ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp);
+ ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old_q);
virtual void do_oop(narrowOop* p) { work(p); }
virtual void do_oop(oop* p) { work(p); }
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp
index edfb62b4046..a9c6a3395eb 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -59,15 +60,18 @@ void ShenandoahSuperClosure::do_nmethod(nmethod* nm) {
// ========= Marking
//
-ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
+ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q,
+ ShenandoahReferenceProcessor* rp,
+ ShenandoahObjToScanQueue* old_q) :
ShenandoahSuperClosure(rp),
_queue(q),
+ _old_queue(old_q),
_mark_context(ShenandoahHeap::heap()->marking_context()),
_weak(false) {}
template
inline void ShenandoahMarkRefsSuperClosure::work(T* p) {
- ShenandoahMark::mark_through_ref(p, _queue, _mark_context, _weak);
+ ShenandoahMark::mark_through_ref(p, _queue, _old_queue, _mark_context, _weak);
}
ShenandoahForwardedIsAliveClosure::ShenandoahForwardedIsAliveClosure() :
@@ -79,7 +83,7 @@ bool ShenandoahForwardedIsAliveClosure::do_object_b(oop obj) {
}
obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
shenandoah_assert_not_forwarded_if(nullptr, obj, ShenandoahHeap::heap()->is_concurrent_mark_in_progress());
- return _mark_context->is_marked(obj);
+ return _mark_context->is_marked_or_old(obj);
}
ShenandoahIsAliveClosure::ShenandoahIsAliveClosure() :
@@ -90,7 +94,7 @@ bool ShenandoahIsAliveClosure::do_object_b(oop obj) {
return false;
}
shenandoah_assert_not_forwarded(nullptr, obj);
- return _mark_context->is_marked(obj);
+ return _mark_context->is_marked_or_old(obj);
}
BoolObjectClosure* ShenandoahIsAliveSelector::is_alive_closure() {
@@ -105,7 +109,7 @@ ShenandoahKeepAliveClosure::ShenandoahKeepAliveClosure() :
template
void ShenandoahKeepAliveClosure::do_oop_work(T* p) {
assert(ShenandoahHeap::heap()->is_concurrent_mark_in_progress(), "Only for concurrent marking phase");
- assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
+ assert(ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress() || !ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
T o = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(o)) {
@@ -215,8 +219,10 @@ void ShenandoahNMethodAndDisarmClosure::do_nmethod(nmethod* nm) {
//
template
-ShenandoahMarkUpdateRefsClosure::ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
- ShenandoahMarkRefsSuperClosure(q, rp) {
+ShenandoahMarkUpdateRefsClosure::ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q,
+ ShenandoahReferenceProcessor* rp,
+ ShenandoahObjToScanQueue* old_q) :
+ ShenandoahMarkRefsSuperClosure(q, rp, old_q) {
assert(_heap->is_stw_gc_in_progress(), "Can only be used for STW GC");
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp
index 70401b42461..124ab1958eb 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2016, 2023, Red Hat, Inc. All rights reserved.
* Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,6 +26,7 @@
#include "precompiled.hpp"
+#include "gc/shenandoah/shenandoahAgeCensus.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
@@ -41,9 +43,13 @@ ShenandoahCollectionSet::ShenandoahCollectionSet(ShenandoahHeap* heap, ReservedS
_cset_map(_map_space.base() + ((uintx)heap_base >> _region_size_bytes_shift)),
_biased_cset_map(_map_space.base()),
_heap(heap),
+ _has_old_regions(false),
_garbage(0),
_used(0),
+ _live(0),
_region_count(0),
+ _old_garbage(0),
+ _preselected_regions(nullptr),
_current_index(0) {
// The collection set map is reserved to cover the entire heap *and* zero addresses.
@@ -84,17 +90,35 @@ void ShenandoahCollectionSet::add_region(ShenandoahHeapRegion* r) {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
assert(Thread::current()->is_VM_thread(), "Must be VMThread");
assert(!is_in(r), "Already in collection set");
+ assert(!r->is_humongous(), "Only add regular regions to the collection set");
+
_cset_map[r->index()] = 1;
+ size_t live = r->get_live_data_bytes();
+ size_t garbage = r->garbage();
+ size_t free = r->free();
+ if (r->is_young()) {
+ _young_bytes_to_evacuate += live;
+ _young_available_bytes_collected += free;
+ if (ShenandoahHeap::heap()->mode()->is_generational() && r->age() >= ShenandoahGenerationalHeap::heap()->age_census()->tenuring_threshold()) {
+ _young_bytes_to_promote += live;
+ }
+ } else if (r->is_old()) {
+ _old_bytes_to_evacuate += live;
+ _old_garbage += garbage;
+ }
+
_region_count++;
- _garbage += r->garbage();
+ _has_old_regions |= r->is_old();
+ _garbage += garbage;
_used += r->used();
-
+ _live += live;
// Update the region status too. State transition would be checked internally.
r->make_cset();
}
void ShenandoahCollectionSet::clear() {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
+
Copy::zero_to_bytes(_cset_map, _map_size);
#ifdef ASSERT
@@ -104,10 +128,20 @@ void ShenandoahCollectionSet::clear() {
#endif
_garbage = 0;
+ _old_garbage = 0;
_used = 0;
+ _live = 0;
_region_count = 0;
_current_index = 0;
+
+ _young_bytes_to_evacuate = 0;
+ _young_bytes_to_promote = 0;
+ _old_bytes_to_evacuate = 0;
+
+ _young_available_bytes_collected = 0;
+
+ _has_old_regions = false;
}
ShenandoahHeapRegion* ShenandoahCollectionSet::claim_next() {
@@ -151,7 +185,11 @@ ShenandoahHeapRegion* ShenandoahCollectionSet::next() {
}
void ShenandoahCollectionSet::print_on(outputStream* out) const {
- out->print_cr("Collection Set : " SIZE_FORMAT "", count());
+ out->print_cr("Collection Set: Regions: "
+ SIZE_FORMAT ", Garbage: " SIZE_FORMAT "%s, Live: " SIZE_FORMAT "%s, Used: " SIZE_FORMAT "%s", count(),
+ byte_size_in_proper_unit(garbage()), proper_unit_for_byte_size(garbage()),
+ byte_size_in_proper_unit(live()), proper_unit_for_byte_size(live()),
+ byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used()));
debug_only(size_t regions = 0;)
for (size_t index = 0; index < _heap->num_regions(); index ++) {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp
index 8ac2d9fb2ea..ae1971f30d6 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2016, 2020, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -33,6 +34,14 @@
class ShenandoahCollectionSet : public CHeapObj {
friend class ShenandoahHeap;
+ friend class ShenandoahCollectionSetPreselector;
+
+ void establish_preselected(bool *preselected) {
+ assert(_preselected_regions == nullptr, "Over-writing");
+ _preselected_regions = preselected;
+ }
+ void abandon_preselected() { _preselected_regions = nullptr; }
+
private:
size_t const _map_size;
size_t const _region_size_bytes_shift;
@@ -43,10 +52,28 @@ class ShenandoahCollectionSet : public CHeapObj {
ShenandoahHeap* const _heap;
+ bool _has_old_regions;
size_t _garbage;
size_t _used;
+ size_t _live;
size_t _region_count;
+ size_t _young_bytes_to_evacuate;
+ size_t _young_bytes_to_promote;
+ size_t _old_bytes_to_evacuate;
+
+ // How many bytes of old garbage are present in a mixed collection set?
+ size_t _old_garbage;
+
+ // Points to array identifying which tenure-age regions have been preselected
+ // for inclusion in collection set. This field is only valid during brief
+ // spans of time while collection set is being constructed.
+ bool* _preselected_regions;
+
+ // When a region having memory available to be allocated is added to the collection set, the region's available memory
+ // should be subtracted from what's available.
+ size_t _young_available_bytes_collected;
+
shenandoah_padding(0);
volatile size_t _current_index;
shenandoah_padding(1);
@@ -77,8 +104,31 @@ class ShenandoahCollectionSet : public CHeapObj {
void print_on(outputStream* out) const;
- size_t used() const { return _used; }
- size_t garbage() const { return _garbage; }
+ // It is not known how many of these bytes will be promoted.
+ inline size_t get_young_bytes_reserved_for_evacuation();
+ inline size_t get_old_bytes_reserved_for_evacuation();
+
+ inline size_t get_young_bytes_to_be_promoted();
+
+ size_t get_young_available_bytes_collected() { return _young_available_bytes_collected; }
+
+ inline size_t get_old_garbage();
+
+ bool is_preselected(size_t region_idx) {
+ assert(_preselected_regions != nullptr, "Missing etsablish after abandon");
+ return _preselected_regions[region_idx];
+ }
+
+ bool* preselected_regions() {
+ assert(_preselected_regions != nullptr, "Null ptr");
+ return _preselected_regions;
+ }
+
+ bool has_old_regions() const { return _has_old_regions; }
+ size_t used() const { return _used; }
+ size_t live() const { return _live; }
+ size_t garbage() const { return _garbage; }
+
void clear();
private:
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp
index cceebae0b1c..791e9c73b28 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2017, 2020, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -53,4 +54,20 @@ bool ShenandoahCollectionSet::is_in_loc(void* p) const {
return _biased_cset_map[index] == 1;
}
+size_t ShenandoahCollectionSet::get_old_bytes_reserved_for_evacuation() {
+ return _old_bytes_to_evacuate;
+}
+
+size_t ShenandoahCollectionSet::get_young_bytes_reserved_for_evacuation() {
+ return _young_bytes_to_evacuate - _young_bytes_to_promote;
+}
+
+size_t ShenandoahCollectionSet::get_young_bytes_to_be_promoted() {
+ return _young_bytes_to_promote;
+}
+
+size_t ShenandoahCollectionSet::get_old_garbage() {
+ return _old_garbage;
+}
+
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHCOLLECTIONSET_INLINE_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSetPreselector.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSetPreselector.hpp
new file mode 100644
index 00000000000..b78259dd85b
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSetPreselector.hpp
@@ -0,0 +1,51 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHCOLLECTIONSETPRESELECTOR_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHCOLLECTIONSETPRESELECTOR_HPP
+
+#include "gc/shenandoah/shenandoahCollectionSet.hpp"
+#include "memory/resourceArea.hpp"
+
+class ShenandoahCollectionSetPreselector : public StackObj {
+ ShenandoahCollectionSet* _cset;
+ bool* _pset;
+ ResourceMark _rm;
+
+public:
+ ShenandoahCollectionSetPreselector(ShenandoahCollectionSet* cset, size_t num_regions):
+ _cset(cset) {
+ _pset = NEW_RESOURCE_ARRAY(bool, num_regions);
+ for (unsigned int i = 0; i < num_regions; i++) {
+ _pset[i] = false;
+ }
+ _cset->establish_preselected(_pset);
+ }
+
+ ~ShenandoahCollectionSetPreselector() {
+ _cset->abandon_preselected();
+ }
+};
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHCOLLECTIONSETPRESELECTOR_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp
index d14bcc39f45..42500ae8778 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -36,6 +37,10 @@ ShenandoahCollectorPolicy::ShenandoahCollectorPolicy() :
_abbreviated_degenerated_gcs(0),
_success_full_gcs(0),
_consecutive_degenerated_gcs(0),
+ _consecutive_young_gcs(0),
+ _mixed_gcs(0),
+ _success_old_gcs(0),
+ _interrupted_old_gcs(0),
_alloc_failure_degenerated(0),
_alloc_failure_degenerated_upgrade_to_full(0),
_alloc_failure_full(0) {
@@ -66,7 +71,9 @@ void ShenandoahCollectorPolicy::record_degenerated_upgrade_to_full() {
_alloc_failure_degenerated_upgrade_to_full++;
}
-void ShenandoahCollectorPolicy::record_success_concurrent(bool is_abbreviated) {
+void ShenandoahCollectorPolicy::record_success_concurrent(bool is_young, bool is_abbreviated) {
+ update_young(is_young);
+
_consecutive_degenerated_gcs = 0;
_success_concurrent_gcs++;
if (is_abbreviated) {
@@ -74,7 +81,23 @@ void ShenandoahCollectorPolicy::record_success_concurrent(bool is_abbreviated) {
}
}
-void ShenandoahCollectorPolicy::record_success_degenerated(bool is_abbreviated) {
+void ShenandoahCollectorPolicy::record_mixed_cycle() {
+ _mixed_gcs++;
+}
+
+void ShenandoahCollectorPolicy::record_success_old() {
+ _consecutive_young_gcs = 0;
+ _success_old_gcs++;
+}
+
+void ShenandoahCollectorPolicy::record_interrupted_old() {
+ _consecutive_young_gcs = 0;
+ _interrupted_old_gcs++;
+}
+
+void ShenandoahCollectorPolicy::record_success_degenerated(bool is_young, bool is_abbreviated) {
+ update_young(is_young);
+
_success_degenerated_gcs++;
_consecutive_degenerated_gcs++;
if (is_abbreviated) {
@@ -82,8 +105,17 @@ void ShenandoahCollectorPolicy::record_success_degenerated(bool is_abbreviated)
}
}
+void ShenandoahCollectorPolicy::update_young(bool is_young) {
+ if (is_young) {
+ _consecutive_young_gcs++;
+ } else {
+ _consecutive_young_gcs = 0;
+ }
+}
+
void ShenandoahCollectorPolicy::record_success_full() {
_consecutive_degenerated_gcs = 0;
+ _consecutive_young_gcs = 0;
_success_full_gcs++;
}
@@ -101,8 +133,9 @@ bool is_explicit_gc(GCCause::Cause cause) {
}
bool is_implicit_gc(GCCause::Cause cause) {
- return cause != GCCause::_allocation_failure
+ return cause != GCCause::_no_gc
&& cause != GCCause::_shenandoah_concurrent_gc
+ && cause != GCCause::_allocation_failure
&& !is_explicit_gc(cause);
}
@@ -120,6 +153,10 @@ bool is_valid_request(GCCause::Cause cause) {
}
#endif
+bool ShenandoahCollectorPolicy::is_requested_gc(GCCause::Cause cause) {
+ return is_explicit_gc(cause) || is_implicit_gc(cause);
+}
+
bool ShenandoahCollectorPolicy::should_run_full_gc(GCCause::Cause cause) {
return is_explicit_gc(cause) ? !ExplicitGCInvokesConcurrent : !ShenandoahImplicitGCInvokesConcurrent;
}
@@ -141,7 +178,7 @@ void ShenandoahCollectorPolicy::print_gc_stats(outputStream* out) const {
out->print_cr("enough regions with no live objects to skip evacuation.");
out->cr();
- size_t completed_gcs = _success_full_gcs + _success_degenerated_gcs + _success_concurrent_gcs;
+ size_t completed_gcs = _success_full_gcs + _success_degenerated_gcs + _success_concurrent_gcs + _success_old_gcs;
out->print_cr(SIZE_FORMAT_W(5) " Completed GCs", completed_gcs);
size_t explicit_requests = 0;
@@ -171,6 +208,13 @@ void ShenandoahCollectorPolicy::print_gc_stats(outputStream* out) const {
out->print_cr(" " SIZE_FORMAT_W(5) " abbreviated (%.2f%%)", _abbreviated_concurrent_gcs, percent_of(_abbreviated_concurrent_gcs, _success_concurrent_gcs));
out->cr();
+ if (ShenandoahHeap::heap()->mode()->is_generational()) {
+ out->print_cr(SIZE_FORMAT_W(5) " Completed Old GCs (%.2f%%)", _success_old_gcs, percent_of(_success_old_gcs, completed_gcs));
+ out->print_cr(" " SIZE_FORMAT_W(5) " mixed", _mixed_gcs);
+ out->print_cr(" " SIZE_FORMAT_W(5) " interruptions", _interrupted_old_gcs);
+ out->cr();
+ }
+
size_t degenerated_gcs = _alloc_failure_degenerated_upgrade_to_full + _success_degenerated_gcs;
out->print_cr(SIZE_FORMAT_W(5) " Degenerated GCs (%.2f%%)", degenerated_gcs, percent_of(degenerated_gcs, completed_gcs));
out->print_cr(" " SIZE_FORMAT_W(5) " upgraded to Full GC (%.2f%%)", _alloc_failure_degenerated_upgrade_to_full, percent_of(_alloc_failure_degenerated_upgrade_to_full, degenerated_gcs));
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp
index 638acce1456..2c92d91ac99 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,14 +29,10 @@
#include "gc/shared/gcTrace.hpp"
#include "gc/shenandoah/shenandoahGC.hpp"
#include "gc/shenandoah/shenandoahSharedVariables.hpp"
+#include "gc/shenandoah/shenandoahTrace.hpp"
#include "memory/allocation.hpp"
#include "utilities/ostream.hpp"
-class ShenandoahTracer : public GCTracer, public CHeapObj {
-public:
- ShenandoahTracer() : GCTracer(Shenandoah) {}
-};
-
class ShenandoahCollectorPolicy : public CHeapObj {
private:
size_t _success_concurrent_gcs;
@@ -45,6 +42,10 @@ class ShenandoahCollectorPolicy : public CHeapObj {
// Written by control thread, read by mutators
volatile size_t _success_full_gcs;
uint _consecutive_degenerated_gcs;
+ volatile size_t _consecutive_young_gcs;
+ size_t _mixed_gcs;
+ size_t _success_old_gcs;
+ size_t _interrupted_old_gcs;
size_t _alloc_failure_degenerated;
size_t _alloc_failure_degenerated_upgrade_to_full;
size_t _alloc_failure_full;
@@ -58,13 +59,17 @@ class ShenandoahCollectorPolicy : public CHeapObj {
public:
ShenandoahCollectorPolicy();
+ void record_mixed_cycle();
+ void record_success_old();
+ void record_interrupted_old();
+
// A collection cycle may be "abbreviated" if Shenandoah finds a sufficient percentage
// of regions that contain no live objects (ShenandoahImmediateThreshold). These cycles
// end after final mark, skipping the evacuation and reference-updating phases. Such
// cycles are very efficient and are worth tracking. Note that both degenerated and
// concurrent cycles can be abbreviated.
- void record_success_concurrent(bool is_abbreviated);
- void record_success_degenerated(bool is_abbreviated);
+ void record_success_concurrent(bool is_young, bool is_abbreviated);
+ void record_success_degenerated(bool is_young, bool is_abbreviated);
void record_success_full();
void record_alloc_failure_to_degenerated(ShenandoahGC::ShenandoahDegenPoint point);
void record_alloc_failure_to_full();
@@ -89,8 +94,16 @@ class ShenandoahCollectorPolicy : public CHeapObj {
return _consecutive_degenerated_gcs;
}
+ static bool is_requested_gc(GCCause::Cause cause);
static bool should_run_full_gc(GCCause::Cause cause);
static bool should_handle_requested_gc(GCCause::Cause cause);
+
+ inline size_t consecutive_young_gc_count() const {
+ return _consecutive_young_gcs;
+ }
+
+private:
+ void update_young(bool is_young);
};
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHCOLLECTORPOLICY_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp
index d801dda372e..a2f15140991 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -33,6 +34,10 @@
#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc/shenandoah/shenandoahConcurrentGC.hpp"
#include "gc/shenandoah/shenandoahFreeSet.hpp"
+#include "gc/shenandoah/shenandoahGeneration.hpp"
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
+#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
#include "gc/shenandoah/shenandoahLock.hpp"
#include "gc/shenandoah/shenandoahMark.inline.hpp"
#include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
@@ -86,22 +91,21 @@ class ShenandoahBreakpointMarkScope : public StackObj {
}
};
-ShenandoahConcurrentGC::ShenandoahConcurrentGC() :
- _mark(),
+ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) :
+ _mark(generation),
+ _generation(generation),
_degen_point(ShenandoahDegenPoint::_degenerated_unset),
- _abbreviated(false) {
+ _abbreviated(false),
+ _do_old_gc_bootstrap(do_old_gc_bootstrap) {
}
ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
return _degen_point;
}
-void ShenandoahConcurrentGC::cancel() {
- ShenandoahConcurrentMark::cancel();
-}
-
bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
+
ShenandoahBreakpointGCScope breakpoint_gc_scope(cause);
// Reset for upcoming marking
@@ -112,9 +116,18 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
{
ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause);
+
+ // Reset task queue stats here, rather than in mark_concurrent_roots,
+ // because remembered set scan will `push` oops into the queues and
+ // resetting after this happens will lose those counts.
+ TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats());
+
+ // Concurrent remembered set scanning
+ entry_scan_remembered_set();
+
// Concurrent mark roots
entry_mark_roots();
- if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_outside_cycle)) {
+ if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_roots)) {
return false;
}
@@ -132,7 +145,7 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
// in the marking phase and must resume the degenerated cycle from there. If the GC was cancelled
// after final mark, then we've entered the evacuation phase and must resume the degenerated cycle
// from that phase.
- if (heap->is_concurrent_mark_in_progress()) {
+ if (_generation->is_concurrent_mark_in_progress()) {
bool cancelled = check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark);
assert(cancelled, "GC must have been cancelled between concurrent and final mark");
return false;
@@ -150,7 +163,8 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
}
// Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
- // the space. This would be the last action if there is nothing to evacuate.
+ // the space. This would be the last action if there is nothing to evacuate. Note that
+ // we will not age young-gen objects in the case that we skip evacuation.
entry_cleanup_early();
heap->free_set()->log_status_under_lock();
@@ -196,10 +210,32 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
// Update references freed up collection set, kick the cleanup to reclaim the space.
entry_cleanup_complete();
} else {
+ // We chose not to evacuate because we found sufficient immediate garbage.
+ // However, there may still be regions to promote in place, so do that now.
+ if (has_in_place_promotions(heap)) {
+ entry_promote_in_place();
+
+ // If the promote-in-place operation was cancelled, we can have the degenerated
+ // cycle complete the operation. It will see that no evacuations are in progress,
+ // and that there are regions wanting promotion. The risk with not handling the
+ // cancellation would be failing to restore top for these regions and leaving
+ // them unable to serve allocations for the old generation.
+ if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
+ return false;
+ }
+ }
+
+ // At this point, the cycle is effectively complete. If the cycle has been cancelled here,
+ // the control thread will detect it on its next iteration and run a degenerated young cycle.
vmop_entry_final_roots();
_abbreviated = true;
}
+ // We defer generation resizing actions until after cset regions have been recycled. We do this even following an
+ // abbreviated cycle.
+ if (heap->mode()->is_generational()) {
+ ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle();
+ }
return true;
}
@@ -300,7 +336,7 @@ void ShenandoahConcurrentGC::entry_final_updaterefs() {
}
void ShenandoahConcurrentGC::entry_final_roots() {
- static const char* msg = "Pause Final Roots";
+ const char* msg = final_roots_event_message();
ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
EventMark em("%s", msg);
@@ -309,17 +345,47 @@ void ShenandoahConcurrentGC::entry_final_roots() {
void ShenandoahConcurrentGC::entry_reset() {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
+ heap->try_inject_alloc_failure();
+
TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
- static const char* msg = "Concurrent reset";
- ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
- EventMark em("%s", msg);
+ {
+ const char* msg = conc_reset_event_message();
+ ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
+ EventMark em("%s", msg);
+
+ ShenandoahWorkerScope scope(heap->workers(),
+ ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
+ msg);
+ op_reset();
+ }
- ShenandoahWorkerScope scope(heap->workers(),
- ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
- "concurrent reset");
+ if (_do_old_gc_bootstrap) {
+ static const char* msg = "Concurrent reset (Old)";
+ ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset_old);
+ ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
+ ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
+ msg);
+ EventMark em("%s", msg);
- heap->try_inject_alloc_failure();
- op_reset();
+ heap->old_generation()->prepare_gc();
+ }
+}
+
+void ShenandoahConcurrentGC::entry_scan_remembered_set() {
+ if (_generation->is_young()) {
+ ShenandoahHeap* const heap = ShenandoahHeap::heap();
+ TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
+ const char* msg = "Concurrent remembered set scanning";
+ ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::init_scan_rset);
+ EventMark em("%s", msg);
+
+ ShenandoahWorkerScope scope(heap->workers(),
+ ShenandoahWorkerPolicy::calc_workers_for_rs_scanning(),
+ msg);
+
+ heap->try_inject_alloc_failure();
+ _generation->scan_remembered_set(true /* is_concurrent */);
+ }
}
void ShenandoahConcurrentGC::entry_mark_roots() {
@@ -368,7 +434,7 @@ void ShenandoahConcurrentGC::entry_thread_roots() {
void ShenandoahConcurrentGC::entry_weak_refs() {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
- static const char* msg = "Concurrent weak references";
+ const char* msg = conc_weak_refs_event_message();
ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs);
EventMark em("%s", msg);
@@ -383,7 +449,7 @@ void ShenandoahConcurrentGC::entry_weak_refs() {
void ShenandoahConcurrentGC::entry_weak_roots() {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
- static const char* msg = "Concurrent weak roots";
+ const char* msg = conc_weak_roots_event_message();
ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
EventMark em("%s", msg);
@@ -430,7 +496,7 @@ void ShenandoahConcurrentGC::entry_strong_roots() {
void ShenandoahConcurrentGC::entry_cleanup_early() {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
- static const char* msg = "Concurrent cleanup";
+ const char* msg = conc_cleanup_event_message();
ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
EventMark em("%s", msg);
@@ -455,6 +521,23 @@ void ShenandoahConcurrentGC::entry_evacuate() {
op_evacuate();
}
+void ShenandoahConcurrentGC::entry_promote_in_place() {
+ shenandoah_assert_generational();
+
+ ShenandoahHeap* const heap = ShenandoahHeap::heap();
+ TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
+
+ static const char* msg = "Promote in place";
+ ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::promote_in_place);
+ EventMark em("%s", msg);
+
+ ShenandoahWorkerScope scope(heap->workers(),
+ ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
+ "promote in place");
+
+ ShenandoahGenerationalHeap::heap()->promote_regions_in_place(true);
+}
+
void ShenandoahConcurrentGC::entry_update_thread_roots() {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
@@ -486,7 +569,7 @@ void ShenandoahConcurrentGC::entry_updaterefs() {
void ShenandoahConcurrentGC::entry_cleanup_complete() {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
- static const char* msg = "Concurrent cleanup";
+ const char* msg = conc_cleanup_event_message();
ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
EventMark em("%s", msg);
@@ -500,8 +583,7 @@ void ShenandoahConcurrentGC::op_reset() {
if (ShenandoahPacing) {
heap->pacer()->setup_for_reset();
}
-
- heap->prepare_gc();
+ _generation->prepare_gc();
}
class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
@@ -514,7 +596,8 @@ class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionCl
assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
if (r->is_active()) {
// Check if region needs updating its TAMS. We have updated it already during concurrent
- // reset, so it is very likely we don't need to do another write here.
+ // reset, so it is very likely we don't need to do another write here. Since most regions
+ // are not "active", this path is relatively rare.
if (_ctx->top_at_mark_start(r) != r->top()) {
_ctx->capture_top_at_mark_start(r);
}
@@ -536,10 +619,29 @@ void ShenandoahConcurrentGC::op_init_mark() {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
- assert(heap->marking_context()->is_bitmap_clear(), "need clear marking bitmap");
- assert(!heap->marking_context()->is_complete(), "should not be complete");
+ assert(_generation->is_bitmap_clear(), "need clear marking bitmap");
+ assert(!_generation->is_mark_complete(), "should not be complete");
assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
+
+ if (heap->mode()->is_generational()) {
+ if (_generation->is_young()) {
+ // The current implementation of swap_remembered_set() copies the write-card-table to the read-card-table.
+ ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset);
+ _generation->swap_remembered_set();
+ }
+
+ if (_generation->is_global()) {
+ heap->old_generation()->cancel_gc();
+ } else if (heap->is_concurrent_old_mark_in_progress()) {
+ // Purge the SATB buffers, transferring any valid, old pointers to the
+ // old generation mark queue. Any pointers in a young region will be
+ // abandoned.
+ ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_transfer_satb);
+ heap->old_generation()->transfer_pointers_from_satb();
+ }
+ }
+
if (ShenandoahVerify) {
heap->verifier()->verify_before_concmark();
}
@@ -548,18 +650,26 @@ void ShenandoahConcurrentGC::op_init_mark() {
Universe::verify();
}
- heap->set_concurrent_mark_in_progress(true);
+ _generation->set_concurrent_mark_in_progress(true);
start_mark();
- {
+ if (_do_old_gc_bootstrap) {
+ shenandoah_assert_generational();
+ // Update region state for both young and old regions
ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
ShenandoahInitMarkUpdateRegionStateClosure cl;
heap->parallel_heap_region_iterate(&cl);
+ heap->old_generation()->ref_processor()->reset_thread_locals();
+ } else {
+ // Update region state for only young regions
+ ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
+ ShenandoahInitMarkUpdateRegionStateClosure cl;
+ _generation->parallel_heap_region_iterate(&cl);
}
// Weak reference processing
- ShenandoahReferenceProcessor* rp = heap->ref_processor();
+ ShenandoahReferenceProcessor* rp = _generation->ref_processor();
rp->reset_thread_locals();
rp->set_soft_reference_policy(heap->soft_ref_policy()->should_clear_all_soft_refs());
@@ -599,12 +709,22 @@ void ShenandoahConcurrentGC::op_final_mark() {
// Notify JVMTI that the tagmap table will need cleaning.
JvmtiTagMap::set_needs_cleaning();
- heap->prepare_regions_and_collection_set(true /*concurrent*/);
+ // The collection set is chosen by prepare_regions_and_collection_set(). Additionally, certain parameters have been
+ // established to govern the evacuation efforts that are about to begin. Refer to comments on reserve members in
+ // ShenandoahGeneration and ShenandoahOldGeneration for more detail.
+ _generation->prepare_regions_and_collection_set(true /*concurrent*/);
// Has to be done after cset selection
heap->prepare_concurrent_roots();
if (!heap->collection_set()->is_empty()) {
+ LogTarget(Debug, gc, cset) lt;
+ if (lt.is_enabled()) {
+ ResourceMark rm;
+ LogStream ls(lt);
+ heap->collection_set()->print_on(&ls);
+ }
+
if (ShenandoahVerify) {
heap->verifier()->verify_before_evacuation();
}
@@ -622,7 +742,11 @@ void ShenandoahConcurrentGC::op_final_mark() {
}
} else {
if (ShenandoahVerify) {
- heap->verifier()->verify_after_concmark();
+ if (has_in_place_promotions(heap)) {
+ heap->verifier()->verify_after_concmark_with_promotions();
+ } else {
+ heap->verifier()->verify_after_concmark();
+ }
}
if (VerifyAfterGC) {
@@ -632,39 +756,47 @@ void ShenandoahConcurrentGC::op_final_mark() {
}
}
+bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
+ return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
+}
+
+template
class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
private:
OopClosure* const _oops;
-
public:
- ShenandoahConcurrentEvacThreadClosure(OopClosure* oops);
- void do_thread(Thread* thread);
-};
+ explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
-ShenandoahConcurrentEvacThreadClosure::ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) :
- _oops(oops) {
-}
-
-void ShenandoahConcurrentEvacThreadClosure::do_thread(Thread* thread) {
- JavaThread* const jt = JavaThread::cast(thread);
- StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
-}
+ void do_thread(Thread* thread) override {
+ JavaThread* const jt = JavaThread::cast(thread);
+ StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
+ if (GENERATIONAL) {
+ ShenandoahThreadLocalData::enable_plab_promotions(thread);
+ }
+ }
+};
+template
class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask {
private:
ShenandoahJavaThreadsIterator _java_threads;
public:
- ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
+ explicit ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
_java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
}
- void work(uint worker_id) {
+ void work(uint worker_id) override {
+ if (GENERATIONAL) {
+ Thread* worker_thread = Thread::current();
+ ShenandoahThreadLocalData::enable_plab_promotions(worker_thread);
+ }
+
// ShenandoahEvacOOMScope has to be setup by ShenandoahContextEvacuateUpdateRootsClosure.
// Otherwise, may deadlock with watermark lock
ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
- ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
+ ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
_java_threads.threads_do(&thr_cl, worker_id);
}
};
@@ -673,8 +805,13 @@ void ShenandoahConcurrentGC::op_thread_roots() {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
assert(heap->is_evacuation_in_progress(), "Checked by caller");
ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
- ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers());
- heap->workers()->run_task(&task);
+ if (heap->mode()->is_generational()) {
+ ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers());
+ heap->workers()->run_task(&task);
+ } else {
+ ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers());
+ heap->workers()->run_task(&task);
+ }
}
void ShenandoahConcurrentGC::op_weak_refs() {
@@ -685,7 +822,7 @@ void ShenandoahConcurrentGC::op_weak_refs() {
if (heap->gc_cause() == GCCause::_wb_breakpoint) {
ShenandoahBreakpoint::at_after_reference_processing_started();
}
- heap->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
+ _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
}
class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
@@ -712,8 +849,11 @@ void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
const oop obj = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(obj)) {
if (!_mark_context->is_marked(obj)) {
- // Note: The obj is dead here. Do not touch it, just clear.
- ShenandoahHeap::atomic_clear_oop(p, obj);
+ shenandoah_assert_generations_reconciled();
+ if (_heap->is_in_active_generation(obj)) {
+ // Note: The obj is dead here. Do not touch it, just clear.
+ ShenandoahHeap::atomic_clear_oop(p, obj);
+ }
} else if (_evac_in_progress && _heap->in_collection_set(obj)) {
oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
if (resolved == obj) {
@@ -819,6 +959,9 @@ void ShenandoahConcurrentGC::op_weak_roots() {
ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
heap->rendezvous_threads("Shenandoah Concurrent Weak Roots");
}
+ // We can only toggle concurrent_weak_root_in_progress flag
+ // at a safepoint, so that mutators see a consistent
+ // value. The flag will be cleared at the next safepoint.
}
void ShenandoahConcurrentGC::op_class_unloading() {
@@ -915,11 +1058,10 @@ void ShenandoahConcurrentGC::op_init_updaterefs() {
heap->set_evacuation_in_progress(false);
heap->set_concurrent_weak_root_in_progress(false);
heap->prepare_update_heap_references(true /*concurrent*/);
+ heap->set_update_refs_in_progress(true);
if (ShenandoahVerify) {
heap->verifier()->verify_before_updaterefs();
}
-
- heap->set_update_refs_in_progress(true);
if (ShenandoahPacing) {
heap->pacer()->setup_for_updaterefs();
}
@@ -967,7 +1109,7 @@ void ShenandoahConcurrentGC::op_final_updaterefs() {
// Clear cancelled GC, if set. On cancellation path, the block before would handle
// everything.
if (heap->cancelled_gc()) {
- heap->clear_cancelled_gc();
+ heap->clear_cancelled_gc(true /* clear oom handler */);
}
// Has to be done before cset is clear
@@ -975,11 +1117,35 @@ void ShenandoahConcurrentGC::op_final_updaterefs() {
heap->verifier()->verify_roots_in_to_space();
}
+ // If we are running in generational mode and this is an aging cycle, this will also age active
+ // regions that haven't been used for allocation.
heap->update_heap_region_states(true /*concurrent*/);
heap->set_update_refs_in_progress(false);
heap->set_has_forwarded_objects(false);
+ if (heap->mode()->is_generational() && heap->is_concurrent_old_mark_in_progress()) {
+ // When the SATB barrier is left on to support concurrent old gen mark, it may pick up writes to
+ // objects in the collection set. After those objects are evacuated, the pointers in the
+ // SATB are no longer safe. Once we have finished update references, we are guaranteed that
+ // no more writes to the collection set are possible.
+ //
+ // This will transfer any old pointers in _active_ regions from the SATB to the old gen
+ // mark queues. All other pointers will be discarded. This would also discard any pointers
+ // in old regions that were included in a mixed evacuation. We aren't using the SATB filter
+ // methods here because we cannot control when they execute. If the SATB filter runs _after_
+ // a region has been recycled, we will not be able to detect the bad pointer.
+ //
+ // We are not concerned about skipping this step in abbreviated cycles because regions
+ // with no live objects cannot have been written to and so cannot have entries in the SATB
+ // buffers.
+ heap->old_generation()->transfer_pointers_from_satb();
+
+ // Aging_cycle is only relevant during evacuation cycle for individual objects and during final mark for
+ // entire regions. Both of these relevant operations occur before final update refs.
+ ShenandoahGenerationalHeap::heap()->set_aging_cycle(false);
+ }
+
if (ShenandoahVerify) {
heap->verifier()->verify_after_updaterefs();
}
@@ -992,7 +1158,23 @@ void ShenandoahConcurrentGC::op_final_updaterefs() {
}
void ShenandoahConcurrentGC::op_final_roots() {
- ShenandoahHeap::heap()->set_concurrent_weak_root_in_progress(false);
+
+ ShenandoahHeap *heap = ShenandoahHeap::heap();
+ heap->set_concurrent_weak_root_in_progress(false);
+ heap->set_evacuation_in_progress(false);
+
+ if (heap->mode()->is_generational()) {
+ // If the cycle was shortened for having enough immediate garbage, this could be
+ // the last GC safepoint before concurrent marking of old resumes. We must be sure
+ // that old mark threads don't see any pointers to garbage in the SATB buffers.
+ if (heap->is_concurrent_old_mark_in_progress()) {
+ heap->old_generation()->transfer_pointers_from_satb();
+ }
+
+ if (!_generation->is_old()) {
+ ShenandoahGenerationalHeap::heap()->update_region_ages(_generation->complete_marking_context());
+ }
+ }
}
void ShenandoahConcurrentGC::op_cleanup_complete() {
@@ -1011,28 +1193,71 @@ const char* ShenandoahConcurrentGC::init_mark_event_message() const {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
if (heap->unload_classes()) {
- return "Pause Init Mark (unload classes)";
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", " (unload classes)");
} else {
- return "Pause Init Mark";
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", "");
}
}
const char* ShenandoahConcurrentGC::final_mark_event_message() const {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
- assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
+ assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
+ "Should not have forwarded objects during final mark, unless old gen concurrent mark is running");
+
if (heap->unload_classes()) {
- return "Pause Final Mark (unload classes)";
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", " (unload classes)");
} else {
- return "Pause Final Mark";
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", "");
}
}
const char* ShenandoahConcurrentGC::conc_mark_event_message() const {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
- assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
+ assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
+ "Should not have forwarded objects concurrent mark, unless old gen concurrent mark is running");
if (heap->unload_classes()) {
- return "Concurrent marking (unload classes)";
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", " (unload classes)");
+ } else {
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", "");
+ }
+}
+
+const char* ShenandoahConcurrentGC::conc_reset_event_message() const {
+ if (ShenandoahHeap::heap()->unload_classes()) {
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", " (unload classes)");
+ } else {
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", "");
+ }
+}
+
+const char* ShenandoahConcurrentGC::final_roots_event_message() const {
+ if (ShenandoahHeap::heap()->unload_classes()) {
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", " (unload classes)");
+ } else {
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", "");
+ }
+}
+
+const char* ShenandoahConcurrentGC::conc_weak_refs_event_message() const {
+ if (ShenandoahHeap::heap()->unload_classes()) {
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", " (unload classes)");
+ } else {
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", "");
+ }
+}
+
+const char* ShenandoahConcurrentGC::conc_weak_roots_event_message() const {
+ if (ShenandoahHeap::heap()->unload_classes()) {
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", " (unload classes)");
+ } else {
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", "");
+ }
+}
+
+const char* ShenandoahConcurrentGC::conc_cleanup_event_message() const {
+ if (ShenandoahHeap::heap()->unload_classes()) {
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", " (unload classes)");
} else {
- return "Concurrent marking";
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", "");
}
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp
index 4cd1a841350..a1837068a7c 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -30,6 +31,8 @@
#include "gc/shenandoah/shenandoahGC.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
+class ShenandoahGeneration;
+
class VM_ShenandoahInitMark;
class VM_ShenandoahFinalMarkStartEvac;
class VM_ShenandoahInitUpdateRefs;
@@ -42,22 +45,24 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
friend class VM_ShenandoahFinalUpdateRefs;
friend class VM_ShenandoahFinalRoots;
+protected:
+ ShenandoahConcurrentMark _mark;
+ ShenandoahGeneration* const _generation;
+
private:
- ShenandoahConcurrentMark _mark;
- ShenandoahDegenPoint _degen_point;
- bool _abbreviated;
+ ShenandoahDegenPoint _degen_point;
+ bool _abbreviated;
+ const bool _do_old_gc_bootstrap;
public:
- ShenandoahConcurrentGC();
- bool collect(GCCause::Cause cause);
+ ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap);
+ bool collect(GCCause::Cause cause) override;
ShenandoahDegenPoint degen_point() const;
// Return true if this cycle found enough immediate garbage to skip evacuation
bool abbreviated() const { return _abbreviated; }
- // Cancel ongoing concurrent GC
- static void cancel();
-private:
+protected:
// Entry points to STW GC operations, these cause a related safepoint, that then
// call the entry method below
void vmop_entry_init_mark();
@@ -78,6 +83,7 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
// for concurrent operation.
void entry_reset();
void entry_mark_roots();
+ void entry_scan_remembered_set();
void entry_mark();
void entry_thread_roots();
void entry_weak_refs();
@@ -90,12 +96,15 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
void entry_updaterefs();
void entry_cleanup_complete();
+ // Called when the collection set is empty, but the generational mode has regions to promote in place
+ void entry_promote_in_place();
+
// Actual work for the phases
void op_reset();
void op_init_mark();
void op_mark_roots();
void op_mark();
- void op_final_mark();
+ virtual void op_final_mark();
void op_thread_roots();
void op_weak_refs();
void op_weak_roots();
@@ -110,16 +119,24 @@ class ShenandoahConcurrentGC : public ShenandoahGC {
void op_final_roots();
void op_cleanup_complete();
+ // Check GC cancellation and abort concurrent GC
+ bool check_cancellation_and_abort(ShenandoahDegenPoint point);
+
+private:
void start_mark();
+ static bool has_in_place_promotions(ShenandoahHeap* heap) ;
+
// Messages for GC trace events, they have to be immortal for
// passing around the logging/tracing systems
const char* init_mark_event_message() const;
const char* final_mark_event_message() const;
+ const char* final_roots_event_message() const;
const char* conc_mark_event_message() const;
-
- // Check GC cancellation and abort concurrent GC
- bool check_cancellation_and_abort(ShenandoahDegenPoint point);
+ const char* conc_reset_event_message() const;
+ const char* conc_weak_refs_event_message() const;
+ const char* conc_weak_roots_event_message() const;
+ const char* conc_cleanup_event_message() const;
};
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHCONCURRENTGC_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp
index 0e2ef4144ad..1709844a8c3 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -30,6 +31,7 @@
#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
#include "gc/shenandoah/shenandoahConcurrentMark.hpp"
+#include "gc/shenandoah/shenandoahGeneration.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahMark.inline.hpp"
#include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
@@ -38,6 +40,7 @@
#include "gc/shenandoah/shenandoahStringDedup.hpp"
#include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
#include "gc/shenandoah/shenandoahUtils.hpp"
+#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
#include "memory/iterator.inline.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/continuation.hpp"
@@ -57,8 +60,13 @@ class ShenandoahConcurrentMarkingTask : public WorkerTask {
void work(uint worker_id) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
ShenandoahConcurrentWorkerSession worker_session(worker_id);
+ ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_mark, ShenandoahPhaseTimings::ParallelMark, worker_id, true);
ShenandoahSuspendibleThreadSetJoiner stsj;
- ShenandoahReferenceProcessor* rp = heap->ref_processor();
+ // Do not use active_generation() : we must use the gc_generation() set by
+ // ShenandoahGCScope on the ControllerThread's stack; no safepoint may
+ // intervene to update active_generation, so we can't
+ // shenandoah_assert_generations_reconciled() here.
+ ShenandoahReferenceProcessor* rp = heap->gc_generation()->ref_processor();
assert(rp != nullptr, "need reference processor");
StringDedup::Requests requests;
_cm->mark_loop(worker_id, _terminator, rp, GENERATION, true /*cancellable*/,
@@ -97,14 +105,16 @@ class ShenandoahFinalMarkingTask : public WorkerTask {
ShenandoahHeap* heap = ShenandoahHeap::heap();
ShenandoahParallelWorkerSession worker_session(worker_id);
- ShenandoahReferenceProcessor* rp = heap->ref_processor();
StringDedup::Requests requests;
+ ShenandoahReferenceProcessor* rp = heap->gc_generation()->ref_processor();
+ shenandoah_assert_generations_reconciled();
// First drain remaining SATB buffers.
{
ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
+ ShenandoahObjToScanQueue* old_q = _cm->get_old_queue(worker_id);
- ShenandoahSATBBufferClosure cl(q);
+ ShenandoahSATBBufferClosure cl(q, old_q);
SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {}
assert(!heap->has_forwarded_objects(), "Not expected");
@@ -119,8 +129,8 @@ class ShenandoahFinalMarkingTask : public WorkerTask {
}
};
-ShenandoahConcurrentMark::ShenandoahConcurrentMark() :
- ShenandoahMark() {}
+ShenandoahConcurrentMark::ShenandoahConcurrentMark(ShenandoahGeneration* generation) :
+ ShenandoahMark(generation) {}
// Mark concurrent roots during concurrent phases
template
@@ -129,10 +139,12 @@ class ShenandoahMarkConcurrentRootsTask : public WorkerTask {
SuspendibleThreadSetJoiner _sts_joiner;
ShenandoahConcurrentRootScanner _root_scanner;
ShenandoahObjToScanQueueSet* const _queue_set;
+ ShenandoahObjToScanQueueSet* const _old_queue_set;
ShenandoahReferenceProcessor* const _rp;
public:
ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
+ ShenandoahObjToScanQueueSet* old,
ShenandoahReferenceProcessor* rp,
ShenandoahPhaseTimings::Phase phase,
uint nworkers);
@@ -141,12 +153,14 @@ class ShenandoahMarkConcurrentRootsTask : public WorkerTask {
template
ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
- ShenandoahReferenceProcessor* rp,
- ShenandoahPhaseTimings::Phase phase,
- uint nworkers) :
+ ShenandoahObjToScanQueueSet* old,
+ ShenandoahReferenceProcessor* rp,
+ ShenandoahPhaseTimings::Phase phase,
+ uint nworkers) :
WorkerTask("Shenandoah Concurrent Mark Roots"),
_root_scanner(nworkers, phase),
_queue_set(qs),
+ _old_queue_set(old),
_rp(rp) {
assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
}
@@ -155,7 +169,9 @@ template
void ShenandoahMarkConcurrentRootsTask::work(uint worker_id) {
ShenandoahConcurrentWorkerSession worker_session(worker_id);
ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id);
- ShenandoahMarkRefsClosure cl(q, _rp);
+ ShenandoahObjToScanQueue* old_q = (_old_queue_set == nullptr) ?
+ nullptr : _old_queue_set->queue(worker_id);
+ ShenandoahMarkRefsClosure cl(q, _rp, old_q);
_root_scanner.roots_do(&cl, worker_id);
}
@@ -163,14 +179,38 @@ void ShenandoahConcurrentMark::mark_concurrent_roots() {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
assert(!heap->has_forwarded_objects(), "Not expected");
- TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
-
WorkerThreads* workers = heap->workers();
- ShenandoahReferenceProcessor* rp = heap->ref_processor();
- task_queues()->reserve(workers->active_workers());
- ShenandoahMarkConcurrentRootsTask task(task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
-
- workers->run_task(&task);
+ ShenandoahReferenceProcessor* rp = _generation->ref_processor();
+ _generation->reserve_task_queues(workers->active_workers());
+ switch (_generation->type()) {
+ case YOUNG: {
+ ShenandoahMarkConcurrentRootsTask task(task_queues(), old_task_queues(), rp,
+ ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
+ workers->run_task(&task);
+ break;
+ }
+ case GLOBAL: {
+ assert(old_task_queues() == nullptr, "Global mark should not have old gen mark queues");
+ ShenandoahMarkConcurrentRootsTask task(task_queues(), nullptr, rp,
+ ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
+ workers->run_task(&task);
+ break;
+ }
+ case NON_GEN: {
+ assert(old_task_queues() == nullptr, "Non-generational mark should not have old gen mark queues");
+ ShenandoahMarkConcurrentRootsTask task(task_queues(), nullptr, rp,
+ ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers());
+ workers->run_task(&task);
+ break;
+ }
+ case OLD: {
+ // We use a YOUNG generation cycle to bootstrap concurrent old marking.
+ ShouldNotReachHere();
+ break;
+ }
+ default:
+ ShouldNotReachHere();
+ }
}
class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure {
@@ -192,12 +232,38 @@ void ShenandoahConcurrentMark::concurrent_mark() {
uint nworkers = workers->active_workers();
task_queues()->reserve(nworkers);
+ ShenandoahGenerationType gen_type = _generation->type();
ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set();
ShenandoahFlushSATBHandshakeClosure flush_satb(qset);
for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) {
- TaskTerminator terminator(nworkers, task_queues());
- ShenandoahConcurrentMarkingTask task(this, &terminator);
- workers->run_task(&task);
+ switch (gen_type) {
+ case YOUNG: {
+ TaskTerminator terminator(nworkers, task_queues());
+ ShenandoahConcurrentMarkingTask task(this, &terminator);
+ workers->run_task(&task);
+ break;
+ }
+ case OLD: {
+ TaskTerminator terminator(nworkers, task_queues());
+ ShenandoahConcurrentMarkingTask task(this, &terminator);
+ workers->run_task(&task);
+ break;
+ }
+ case GLOBAL: {
+ TaskTerminator terminator(nworkers, task_queues());
+ ShenandoahConcurrentMarkingTask task(this, &terminator);
+ workers->run_task(&task);
+ break;
+ }
+ case NON_GEN: {
+ TaskTerminator terminator(nworkers, task_queues());
+ ShenandoahConcurrentMarkingTask task(this, &terminator);
+ workers->run_task(&task);
+ break;
+ }
+ default:
+ ShouldNotReachHere();
+ }
if (heap->cancelled_gc()) {
// GC is cancelled, break out.
@@ -226,9 +292,8 @@ void ShenandoahConcurrentMark::finish_mark() {
assert(task_queues()->is_empty(), "Should be empty");
TASKQUEUE_STATS_ONLY(task_queues()->print_and_reset_taskqueue_stats(""));
- ShenandoahHeap* const heap = ShenandoahHeap::heap();
- heap->set_concurrent_mark_in_progress(false);
- heap->mark_complete_marking_context();
+ _generation->set_concurrent_mark_in_progress(false);
+ _generation->set_mark_complete();
end_mark();
}
@@ -248,15 +313,32 @@ void ShenandoahConcurrentMark::finish_mark_work() {
StrongRootsScope scope(nworkers);
TaskTerminator terminator(nworkers, task_queues());
- ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
- heap->workers()->run_task(&task);
- assert(task_queues()->is_empty(), "Should be empty");
-}
+ switch (_generation->type()) {
+ case YOUNG:{
+ ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
+ heap->workers()->run_task(&task);
+ break;
+ }
+ case OLD:{
+ ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
+ heap->workers()->run_task(&task);
+ break;
+ }
+ case GLOBAL:{
+ ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
+ heap->workers()->run_task(&task);
+ break;
+ }
+ case NON_GEN:{
+ ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
+ heap->workers()->run_task(&task);
+ break;
+ }
+ default:
+ ShouldNotReachHere();
+ }
-void ShenandoahConcurrentMark::cancel() {
- clear();
- ShenandoahReferenceProcessor* rp = ShenandoahHeap::heap()->ref_processor();
- rp->abandon_partial_discovery();
+ assert(task_queues()->is_empty(), "Should be empty");
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp
index b658e42dfb8..17b27037658 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp
@@ -32,22 +32,24 @@ template
class ShenandoahConcurrentMarkingTask;
template
class ShenandoahFinalMarkingTask;
+class ShenandoahGeneration;
class ShenandoahConcurrentMark: public ShenandoahMark {
template friend class ShenandoahConcurrentMarkingTask;
template friend class ShenandoahFinalMarkingTask;
public:
- ShenandoahConcurrentMark();
+ ShenandoahConcurrentMark(ShenandoahGeneration* generation);
+
// Concurrent mark roots
void mark_concurrent_roots();
+
// Concurrent mark
void concurrent_mark();
+
// Finish mark at a safepoint
void finish_mark();
- static void cancel();
-
private:
void finish_mark_work();
};
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp
index df2d6d092e6..8a82498225a 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp
@@ -30,6 +30,7 @@
#include "gc/shenandoah/shenandoahDegeneratedGC.hpp"
#include "gc/shenandoah/shenandoahFreeSet.hpp"
#include "gc/shenandoah/shenandoahFullGC.hpp"
+#include "gc/shenandoah/shenandoahGeneration.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
#include "gc/shenandoah/shenandoahPacer.inline.hpp"
@@ -85,7 +86,7 @@ void ShenandoahControlThread::run_service() {
if (alloc_failure_pending) {
// Allocation failure takes precedence: we have to deal with it first thing
- log_info(gc)("Trigger: Handle Allocation Failure");
+ heuristics->log_trigger("Handle Allocation Failure");
cause = GCCause::_allocation_failure;
@@ -104,7 +105,7 @@ void ShenandoahControlThread::run_service() {
}
} else if (is_gc_requested) {
cause = requested_gc_cause;
- log_info(gc)("Trigger: GC request (%s)", GCCause::to_string(cause));
+ heuristics->log_trigger("GC request (%s)", GCCause::to_string(cause));
heuristics->record_requested_gc();
if (ShenandoahCollectorPolicy::should_run_full_gc(cause)) {
@@ -315,19 +316,21 @@ void ShenandoahControlThread::service_concurrent_normal_cycle(GCCause::Cause cau
if (check_cancellation_or_degen(ShenandoahGC::_degenerated_outside_cycle)) return;
GCIdMark gc_id_mark;
- ShenandoahGCSession session(cause);
+ ShenandoahGCSession session(cause, heap->global_generation());
TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
- ShenandoahConcurrentGC gc;
+ ShenandoahConcurrentGC gc(heap->global_generation(), false);
if (gc.collect(cause)) {
// Cycle is complete. There were no failed allocation requests and no degeneration, so count this as good progress.
heap->notify_gc_progress();
- heap->heuristics()->record_success_concurrent();
- heap->shenandoah_policy()->record_success_concurrent(gc.abbreviated());
+ heap->global_generation()->heuristics()->record_success_concurrent();
+ heap->shenandoah_policy()->record_success_concurrent(false, gc.abbreviated());
+ heap->log_heap_status("At end of GC");
} else {
assert(heap->cancelled_gc(), "Must have been cancelled");
check_cancellation_or_degen(gc.degen_point());
+ heap->log_heap_status("At end of cancelled GC");
}
}
@@ -350,8 +353,9 @@ void ShenandoahControlThread::stop_service() {
}
void ShenandoahControlThread::service_stw_full_cycle(GCCause::Cause cause) {
+ ShenandoahHeap* const heap = ShenandoahHeap::heap();
GCIdMark gc_id_mark;
- ShenandoahGCSession session(cause);
+ ShenandoahGCSession session(cause, heap->global_generation());
ShenandoahFullGC gc;
gc.collect(cause);
@@ -359,11 +363,11 @@ void ShenandoahControlThread::service_stw_full_cycle(GCCause::Cause cause) {
void ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahGC::ShenandoahDegenPoint point) {
assert (point != ShenandoahGC::_degenerated_unset, "Degenerated point should be set");
-
+ ShenandoahHeap* const heap = ShenandoahHeap::heap();
GCIdMark gc_id_mark;
- ShenandoahGCSession session(cause);
+ ShenandoahGCSession session(cause, heap->global_generation());
- ShenandoahDegenGC gc(point);
+ ShenandoahDegenGC gc(point, heap->global_generation());
gc.collect(cause);
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
index 2249c38455f..7d2690ef5f6 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -29,26 +30,38 @@
#include "gc/shenandoah/shenandoahConcurrentMark.hpp"
#include "gc/shenandoah/shenandoahDegeneratedGC.hpp"
#include "gc/shenandoah/shenandoahFullGC.hpp"
+#include "gc/shenandoah/shenandoahGeneration.hpp"
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahMetrics.hpp"
#include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
#include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
#include "gc/shenandoah/shenandoahSTWMark.hpp"
#include "gc/shenandoah/shenandoahUtils.hpp"
#include "gc/shenandoah/shenandoahVerifier.hpp"
+#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
#include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
#include "gc/shenandoah/shenandoahVMOperations.hpp"
#include "runtime/vmThread.hpp"
#include "utilities/events.hpp"
-ShenandoahDegenGC::ShenandoahDegenGC(ShenandoahDegenPoint degen_point) :
+ShenandoahDegenGC::ShenandoahDegenGC(ShenandoahDegenPoint degen_point, ShenandoahGeneration* generation) :
ShenandoahGC(),
_degen_point(degen_point),
+ _generation(generation),
_abbreviated(false) {
}
bool ShenandoahDegenGC::collect(GCCause::Cause cause) {
vmop_degenerated();
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+ if (heap->mode()->is_generational()) {
+ bool is_bootstrap_gc = heap->old_generation()->is_bootstrapping();
+ heap->mmu_tracker()->record_degenerated(GCId::current(), is_bootstrap_gc);
+ const char* msg = is_bootstrap_gc? "At end of Degenerated Bootstrap Old GC": "At end of Degenerated Young GC";
+ heap->log_heap_status(msg);
+ }
return true;
}
@@ -64,7 +77,6 @@ void ShenandoahDegenGC::entry_degenerated() {
ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::degen_gc, true /* log_heap_usage */);
EventMark em("%s", msg);
ShenandoahHeap* const heap = ShenandoahHeap::heap();
-
ShenandoahWorkerScope scope(heap->workers(),
ShenandoahWorkerPolicy::calc_workers_for_stw_degenerated(),
"stw degenerated gc");
@@ -79,7 +91,26 @@ void ShenandoahDegenGC::op_degenerated() {
// Degenerated GC is STW, but it can also fail. Current mechanics communicates
// GC failure via cancelled_concgc() flag. So, if we detect the failure after
// some phase, we have to upgrade the Degenerate GC to Full GC.
- heap->clear_cancelled_gc();
+ heap->clear_cancelled_gc(true /* clear oom handler */);
+
+#ifdef ASSERT
+ if (heap->mode()->is_generational()) {
+ ShenandoahOldGeneration* old_generation = heap->old_generation();
+ if (!heap->is_concurrent_old_mark_in_progress()) {
+ // If we are not marking the old generation, there should be nothing in the old mark queues
+ assert(old_generation->task_queues()->is_empty(), "Old gen task queues should be empty");
+ }
+
+ if (_generation->is_global()) {
+ // If we are in a global cycle, the old generation should not be marking. It is, however,
+ // allowed to be holding regions for evacuation or coalescing.
+ assert(old_generation->is_idle()
+ || old_generation->is_doing_mixed_evacuations()
+ || old_generation->is_preparing_for_mark(),
+ "Old generation cannot be in state: %s", old_generation->state_name());
+ }
+ }
+#endif
ShenandoahMetricsSnapshot metrics;
metrics.snap_before();
@@ -95,17 +126,51 @@ void ShenandoahDegenGC::op_degenerated() {
// space. It makes little sense to wait for Full GC to reclaim as much as it can, when
// we can do the most aggressive degen cycle, which includes processing references and
// class unloading, unless those features are explicitly disabled.
- //
-
- // Degenerated from concurrent root mark, reset the flag for STW mark
- if (heap->is_concurrent_mark_in_progress()) {
- ShenandoahConcurrentMark::cancel();
- heap->set_concurrent_mark_in_progress(false);
- }
// Note that we can only do this for "outside-cycle" degens, otherwise we would risk
// changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
- heap->set_unload_classes(heap->heuristics()->can_unload_classes());
+ heap->set_unload_classes(_generation->heuristics()->can_unload_classes() &&
+ (!heap->mode()->is_generational() || _generation->is_global()));
+
+ if (heap->mode()->is_generational() && _generation->is_young()) {
+ // Swap remembered sets for young
+ _generation->swap_remembered_set();
+ }
+
+ case _degenerated_roots:
+ // Degenerated from concurrent root mark, reset the flag for STW mark
+ if (!heap->mode()->is_generational()) {
+ if (heap->is_concurrent_mark_in_progress()) {
+ heap->cancel_concurrent_mark();
+ }
+ } else {
+ if (_generation->is_concurrent_mark_in_progress()) {
+ // We want to allow old generation marking to be punctuated by young collections
+ // (even if they have degenerated). If this is a global cycle, we'd have cancelled
+ // the entire old gc before coming into this switch. Note that cancel_marking on
+ // the generation does NOT abandon incomplete SATB buffers as cancel_concurrent_mark does.
+ // We need to separate out the old pointers which is done below.
+ _generation->cancel_marking();
+ }
+
+ if (heap->is_concurrent_mark_in_progress()) {
+ // If either old or young marking is in progress, the SATB barrier will be enabled.
+ // The SATB buffer may hold a mix of old and young pointers. The old pointers need to be
+ // transferred to the old generation mark queues and the young pointers are NOT part
+ // of this snapshot, so they must be dropped here. It is safe to drop them here because
+ // we will rescan the roots on this safepoint.
+ heap->old_generation()->transfer_pointers_from_satb();
+ }
+
+ if (_degen_point == ShenandoahDegenPoint::_degenerated_roots) {
+ // We only need this if the concurrent cycle has already swapped the card tables.
+ // Marking will use the 'read' table, but interesting pointers may have been
+ // recorded in the 'write' table in the time between the cancelled concurrent cycle
+ // and this degenerated cycle. These pointers need to be included the 'read' table
+ // used to scan the remembered set during the STW mark which follows here.
+ _generation->merge_write_table();
+ }
+ }
op_reset();
@@ -169,7 +234,6 @@ void ShenandoahDegenGC::op_degenerated() {
{
heap->sync_pinned_region_status();
heap->collection_set()->clear_current_index();
-
ShenandoahHeapRegion* r;
while ((r = heap->collection_set()->next()) != nullptr) {
if (r->is_pinned()) {
@@ -186,8 +250,17 @@ void ShenandoahDegenGC::op_degenerated() {
op_degenerated_fail();
return;
}
+ } else if (has_in_place_promotions(heap)) {
+ // We have nothing to evacuate, but there are still regions to promote in place.
+ ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_promote_regions);
+ ShenandoahGenerationalHeap::heap()->promote_regions_in_place(false /* concurrent*/);
}
+ // Update collector state regardless of whether there are forwarded objects
+ heap->set_evacuation_in_progress(false);
+ heap->set_concurrent_weak_root_in_progress(false);
+ heap->set_concurrent_strong_root_in_progress(false);
+
// If heuristics thinks we should do the cycle, this flag would be set,
// and we need to do update-refs. Otherwise, it would be the shortcut cycle.
if (heap->has_forwarded_objects()) {
@@ -209,6 +282,11 @@ void ShenandoahDegenGC::op_degenerated() {
ShenandoahCodeRoots::disarm_nmethods();
op_cleanup_complete();
+
+ if (heap->mode()->is_generational()) {
+ ShenandoahGenerationalHeap::heap()->complete_degenerated_cycle();
+ }
+
break;
default:
ShouldNotReachHere();
@@ -231,25 +309,24 @@ void ShenandoahDegenGC::op_degenerated() {
op_degenerated_futile();
} else {
heap->notify_gc_progress();
- heap->shenandoah_policy()->record_success_degenerated(_abbreviated);
- heap->heuristics()->record_success_degenerated();
+ heap->shenandoah_policy()->record_success_degenerated(_generation->is_young(), _abbreviated);
+ _generation->heuristics()->record_success_degenerated();
}
}
void ShenandoahDegenGC::op_reset() {
- ShenandoahHeap::heap()->prepare_gc();
+ _generation->prepare_gc();
}
void ShenandoahDegenGC::op_mark() {
- assert(!ShenandoahHeap::heap()->is_concurrent_mark_in_progress(), "Should be reset");
+ assert(!_generation->is_concurrent_mark_in_progress(), "Should be reset");
ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_stw_mark);
- ShenandoahSTWMark mark(false /*full gc*/);
- mark.clear();
+ ShenandoahSTWMark mark(_generation, false /*full gc*/);
mark.mark();
}
void ShenandoahDegenGC::op_finish_mark() {
- ShenandoahConcurrentMark mark;
+ ShenandoahConcurrentMark mark(_generation);
mark.finish_mark();
}
@@ -261,8 +338,9 @@ void ShenandoahDegenGC::op_prepare_evacuation() {
// STW cleanup weak roots and unload classes
heap->parallel_cleaning(false /*full gc*/);
+
// Prepare regions and collection set
- heap->prepare_regions_and_collection_set(false /*concurrent*/);
+ _generation->prepare_regions_and_collection_set(false /*concurrent*/);
// Retire the TLABs, which will force threads to reacquire their TLABs after the pause.
// This is needed for two reasons. Strong one: new allocations would be with new freeset,
@@ -283,7 +361,11 @@ void ShenandoahDegenGC::op_prepare_evacuation() {
heap->set_has_forwarded_objects(true);
} else {
if (ShenandoahVerify) {
- heap->verifier()->verify_after_concmark();
+ if (has_in_place_promotions(heap)) {
+ heap->verifier()->verify_after_concmark_with_promotions();
+ } else {
+ heap->verifier()->verify_after_concmark();
+ }
}
if (VerifyAfterGC) {
@@ -292,6 +374,10 @@ void ShenandoahDegenGC::op_prepare_evacuation() {
}
}
+bool ShenandoahDegenGC::has_in_place_promotions(const ShenandoahHeap* heap) const {
+ return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
+}
+
void ShenandoahDegenGC::op_cleanup_early() {
ShenandoahHeap::heap()->recycle_trash();
}
@@ -304,10 +390,6 @@ void ShenandoahDegenGC::op_evacuate() {
void ShenandoahDegenGC::op_init_updaterefs() {
// Evacuation has completed
ShenandoahHeap* const heap = ShenandoahHeap::heap();
- heap->set_evacuation_in_progress(false);
- heap->set_concurrent_weak_root_in_progress(false);
- heap->set_concurrent_strong_root_in_progress(false);
-
heap->prepare_update_heap_references(false /*concurrent*/);
heap->set_update_refs_in_progress(true);
}
@@ -356,18 +438,20 @@ void ShenandoahDegenGC::op_degenerated_futile() {
const char* ShenandoahDegenGC::degen_event_message(ShenandoahDegenPoint point) const {
switch (point) {
case _degenerated_unset:
- return "Pause Degenerated GC ()";
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " ()");
case _degenerated_outside_cycle:
- return "Pause Degenerated GC (Outside of Cycle)";
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (Outside of Cycle)");
+ case _degenerated_roots:
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (Roots)");
case _degenerated_mark:
- return "Pause Degenerated GC (Mark)";
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (Mark)");
case _degenerated_evac:
- return "Pause Degenerated GC (Evacuation)";
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (Evacuation)");
case _degenerated_updaterefs:
- return "Pause Degenerated GC (Update Refs)";
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (Update Refs)");
default:
ShouldNotReachHere();
- return "ERROR";
+ SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (?)");
}
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp
index 633c3f1ce73..8dc8ecea75f 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp
@@ -28,15 +28,17 @@
#include "gc/shenandoah/shenandoahGC.hpp"
class VM_ShenandoahDegeneratedGC;
+class ShenandoahGeneration;
class ShenandoahDegenGC : public ShenandoahGC {
friend class VM_ShenandoahDegeneratedGC;
private:
const ShenandoahDegenPoint _degen_point;
+ ShenandoahGeneration* _generation;
bool _abbreviated;
public:
- ShenandoahDegenGC(ShenandoahDegenPoint degen_point);
+ ShenandoahDegenGC(ShenandoahDegenPoint degen_point, ShenandoahGeneration* generation);
bool collect(GCCause::Cause cause);
private:
@@ -64,6 +66,8 @@ class ShenandoahDegenGC : public ShenandoahGC {
void upgrade_to_full();
const char* degen_event_message(ShenandoahDegenPoint point) const;
+
+ bool has_in_place_promotions(const ShenandoahHeap* heap) const;
};
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHDEGENERATEDGC_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacInfo.hpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacInfo.hpp
new file mode 100644
index 00000000000..5ca65763833
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacInfo.hpp
@@ -0,0 +1,120 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHEVACINFO_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHEVACINFO_HPP
+
+#include "memory/allocation.hpp"
+
+class ShenandoahEvacuationInformation : public StackObj {
+ // Values for ShenandoahEvacuationInformation jfr event, sizes stored as bytes
+ size_t _collection_set_regions;
+ size_t _collection_set_used_before;
+ size_t _collection_set_used_after;
+ size_t _collected_old;
+ size_t _collected_promoted;
+ size_t _collected_young;
+ size_t _regions_promoted_humongous;
+ size_t _regions_promoted_regular;
+ size_t _regular_promoted_garbage;
+ size_t _regular_promoted_free;
+ size_t _regions_freed;
+ size_t _regions_immediate;
+ size_t _immediate_size;
+
+public:
+ ShenandoahEvacuationInformation() :
+ _collection_set_regions(0), _collection_set_used_before(0), _collection_set_used_after(0),
+ _collected_old(0), _collected_promoted(0), _collected_young(0), _regions_promoted_humongous(0),
+ _regions_promoted_regular(0), _regular_promoted_garbage(0), _regular_promoted_free(0),
+ _regions_freed(0), _regions_immediate(0), _immediate_size(0) { }
+
+ void set_collection_set_regions(size_t collection_set_regions) {
+ _collection_set_regions = collection_set_regions;
+ }
+
+ void set_collection_set_used_before(size_t used) {
+ _collection_set_used_before = used;
+ }
+
+ void set_collection_set_used_after(size_t used) {
+ _collection_set_used_after = used;
+ }
+
+ void set_collected_old(size_t collected) {
+ _collected_old = collected;
+ }
+
+ void set_collected_promoted(size_t collected) {
+ _collected_promoted = collected;
+ }
+
+ void set_collected_young(size_t collected) {
+ _collected_young = collected;
+ }
+
+ void set_regions_freed(size_t freed) {
+ _regions_freed = freed;
+ }
+
+ void set_regions_promoted_humongous(size_t humongous) {
+ _regions_promoted_humongous = humongous;
+ }
+
+ void set_regions_promoted_regular(size_t regular) {
+ _regions_promoted_regular = regular;
+ }
+
+ void set_regular_promoted_garbage(size_t garbage) {
+ _regular_promoted_garbage = garbage;
+ }
+
+ void set_regular_promoted_free(size_t free) {
+ _regular_promoted_free = free;
+ }
+
+ void set_regions_immediate(size_t immediate) {
+ _regions_immediate = immediate;
+ }
+
+ void set_immediate_size(size_t size) {
+ _immediate_size = size;
+ }
+
+ size_t collection_set_regions() { return _collection_set_regions; }
+ size_t collection_set_used_before() { return _collection_set_used_before; }
+ size_t collection_set_used_after() { return _collection_set_used_after; }
+ size_t collected_old() { return _collected_old; }
+ size_t collected_promoted() { return _collected_promoted; }
+ size_t collected_young() { return _collected_young; }
+ size_t regions_promoted_humongous() { return _regions_promoted_humongous; }
+ size_t regions_promoted_regular() { return _regions_promoted_regular; }
+ size_t regular_promoted_garbage() { return _regular_promoted_garbage; }
+ size_t regular_promoted_free() { return _regular_promoted_free; }
+ size_t regions_freed() { return _regions_freed; }
+ size_t regions_immediate() { return _regions_immediate; }
+ size_t immediate_size() { return _immediate_size; }
+};
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHEVACINFO_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp
new file mode 100644
index 00000000000..ededb99b24e
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp
@@ -0,0 +1,173 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+#include "precompiled.hpp"
+
+#include "gc/shenandoah/shenandoahAgeCensus.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahEvacTracker.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "runtime/threadSMR.inline.hpp"
+#include "runtime/thread.hpp"
+
+ShenandoahEvacuationStats::ShenandoahEvacuationStats()
+ : _evacuations_completed(0), _bytes_completed(0),
+ _evacuations_attempted(0), _bytes_attempted(0),
+ _use_age_table(ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring),
+ _age_table(nullptr) {
+ if (_use_age_table) {
+ _age_table = new AgeTable(false);
+ }
+}
+
+AgeTable* ShenandoahEvacuationStats::age_table() const {
+ assert(_use_age_table, "Don't call");
+ return _age_table;
+}
+
+void ShenandoahEvacuationStats::begin_evacuation(size_t bytes) {
+ ++_evacuations_attempted;
+ _bytes_attempted += bytes;
+}
+
+void ShenandoahEvacuationStats::end_evacuation(size_t bytes) {
+ ++_evacuations_completed;
+ _bytes_completed += bytes;
+}
+
+void ShenandoahEvacuationStats::record_age(size_t bytes, uint age) {
+ assert(_use_age_table, "Don't call!");
+ if (age <= markWord::max_age) { // Filter age sentinel.
+ _age_table->add(age, bytes >> LogBytesPerWord);
+ }
+}
+
+void ShenandoahEvacuationStats::accumulate(const ShenandoahEvacuationStats* other) {
+ _evacuations_completed += other->_evacuations_completed;
+ _bytes_completed += other->_bytes_completed;
+ _evacuations_attempted += other->_evacuations_attempted;
+ _bytes_attempted += other->_bytes_attempted;
+ if (_use_age_table) {
+ _age_table->merge(other->age_table());
+ }
+}
+
+void ShenandoahEvacuationStats::reset() {
+ _evacuations_completed = _evacuations_attempted = 0;
+ _bytes_completed = _bytes_attempted = 0;
+ if (_use_age_table) {
+ _age_table->clear();
+ }
+}
+
+void ShenandoahEvacuationStats::print_on(outputStream* st) {
+#ifndef PRODUCT
+ size_t abandoned_size = _bytes_attempted - _bytes_completed;
+ size_t abandoned_count = _evacuations_attempted - _evacuations_completed;
+ st->print_cr("Evacuated " SIZE_FORMAT "%s across " SIZE_FORMAT " objects, "
+ "abandoned " SIZE_FORMAT "%s across " SIZE_FORMAT " objects.",
+ byte_size_in_proper_unit(_bytes_completed), proper_unit_for_byte_size(_bytes_completed),
+ _evacuations_completed,
+ byte_size_in_proper_unit(abandoned_size), proper_unit_for_byte_size(abandoned_size),
+ abandoned_count);
+#endif
+ if (_use_age_table) {
+ _age_table->print_on(st);
+ }
+}
+
+void ShenandoahEvacuationTracker::print_global_on(outputStream* st) {
+ print_evacuations_on(st, &_workers_global, &_mutators_global);
+}
+
+void ShenandoahEvacuationTracker::print_evacuations_on(outputStream* st,
+ ShenandoahEvacuationStats* workers,
+ ShenandoahEvacuationStats* mutators) {
+ st->print("Workers: ");
+ workers->print_on(st);
+ st->cr();
+ st->print("Mutators: ");
+ mutators->print_on(st);
+ st->cr();
+
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+
+ AgeTable young_region_ages(false);
+ for (uint i = 0; i < heap->num_regions(); ++i) {
+ ShenandoahHeapRegion* r = heap->get_region(i);
+ if (r->is_young()) {
+ young_region_ages.add(r->age(), r->get_live_data_words());
+ }
+ }
+ st->print("Young regions: ");
+ young_region_ages.print_on(st);
+ st->cr();
+}
+
+class ShenandoahStatAggregator : public ThreadClosure {
+public:
+ ShenandoahEvacuationStats* _target;
+ explicit ShenandoahStatAggregator(ShenandoahEvacuationStats* target) : _target(target) {}
+ void do_thread(Thread* thread) override {
+ ShenandoahEvacuationStats* local = ShenandoahThreadLocalData::evacuation_stats(thread);
+ _target->accumulate(local);
+ local->reset();
+ }
+};
+
+ShenandoahCycleStats ShenandoahEvacuationTracker::flush_cycle_to_global() {
+ ShenandoahEvacuationStats mutators, workers;
+
+ ThreadsListHandle java_threads_iterator;
+ ShenandoahStatAggregator aggregate_mutators(&mutators);
+ java_threads_iterator.list()->threads_do(&aggregate_mutators);
+
+ ShenandoahStatAggregator aggregate_workers(&workers);
+ ShenandoahHeap::heap()->gc_threads_do(&aggregate_workers);
+
+ _mutators_global.accumulate(&mutators);
+ _workers_global.accumulate(&workers);
+
+ if (ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring) {
+ // Ingest mutator & worker collected population vectors into the heap's
+ // global census data, and use it to compute an appropriate tenuring threshold
+ // for use in the next cycle.
+ // The first argument is used for any age 0 cohort population that we may otherwise have
+ // missed during the census. This is non-zero only when census happens at marking.
+ ShenandoahGenerationalHeap::heap()->age_census()->update_census(0, _mutators_global.age_table(), _workers_global.age_table());
+ }
+
+ return {workers, mutators};
+}
+
+void ShenandoahEvacuationTracker::begin_evacuation(Thread* thread, size_t bytes) {
+ ShenandoahThreadLocalData::begin_evacuation(thread, bytes);
+}
+
+void ShenandoahEvacuationTracker::end_evacuation(Thread* thread, size_t bytes) {
+ ShenandoahThreadLocalData::end_evacuation(thread, bytes);
+}
+
+void ShenandoahEvacuationTracker::record_age(Thread* thread, size_t bytes, uint age) {
+ ShenandoahThreadLocalData::record_age(thread, bytes, age);
+}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp
new file mode 100644
index 00000000000..7d195656b11
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp
@@ -0,0 +1,81 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHEVACTRACKER_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHEVACTRACKER_HPP
+
+#include "gc/shared/ageTable.hpp"
+#include "utilities/ostream.hpp"
+
+class ShenandoahEvacuationStats : public CHeapObj {
+private:
+ size_t _evacuations_completed;
+ size_t _bytes_completed;
+ size_t _evacuations_attempted;
+ size_t _bytes_attempted;
+
+ bool _use_age_table;
+ AgeTable* _age_table;
+
+ public:
+ ShenandoahEvacuationStats();
+
+ AgeTable* age_table() const;
+
+ void begin_evacuation(size_t bytes);
+ void end_evacuation(size_t bytes);
+ void record_age(size_t bytes, uint age);
+
+ void print_on(outputStream* st);
+ void accumulate(const ShenandoahEvacuationStats* other);
+ void reset();
+};
+
+struct ShenandoahCycleStats {
+ ShenandoahEvacuationStats workers;
+ ShenandoahEvacuationStats mutators;
+};
+
+class ShenandoahEvacuationTracker : public CHeapObj {
+private:
+
+ ShenandoahEvacuationStats _workers_global;
+ ShenandoahEvacuationStats _mutators_global;
+
+public:
+ ShenandoahEvacuationTracker() = default;
+
+ void begin_evacuation(Thread* thread, size_t bytes);
+ void end_evacuation(Thread* thread, size_t bytes);
+ void record_age(Thread* thread, size_t bytes, uint age);
+
+ void print_global_on(outputStream* st);
+ void print_evacuations_on(outputStream* st,
+ ShenandoahEvacuationStats* workers,
+ ShenandoahEvacuationStats* mutators);
+
+ ShenandoahCycleStats flush_cycle_to_global();
+};
+
+#endif //SHARE_GC_SHENANDOAH_SHENANDOAHEVACTRACKER_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp
index 310cd5b8061..1a29fbbc4a7 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp
@@ -25,10 +25,13 @@
#include "precompiled.hpp"
#include "gc/shared/tlab_globals.hpp"
+#include "gc/shenandoah/shenandoahAffiliation.hpp"
#include "gc/shenandoah/shenandoahFreeSet.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
+#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
#include "gc/shenandoah/shenandoahSimpleBitMap.hpp"
#include "gc/shenandoah/shenandoahSimpleBitMap.inline.hpp"
#include "logging/logStream.hpp"
@@ -40,27 +43,96 @@ static const char* partition_name(ShenandoahFreeSetPartitionId t) {
case ShenandoahFreeSetPartitionId::NotFree: return "NotFree";
case ShenandoahFreeSetPartitionId::Mutator: return "Mutator";
case ShenandoahFreeSetPartitionId::Collector: return "Collector";
+ case ShenandoahFreeSetPartitionId::OldCollector: return "OldCollector";
default:
ShouldNotReachHere();
return "Unrecognized";
}
}
+class ShenandoahLeftRightIterator {
+private:
+ idx_t _idx;
+ idx_t _end;
+ ShenandoahRegionPartitions* _partitions;
+ ShenandoahFreeSetPartitionId _partition;
+public:
+ explicit ShenandoahLeftRightIterator(ShenandoahRegionPartitions* partitions, ShenandoahFreeSetPartitionId partition, bool use_empty = false)
+ : _idx(0), _end(0), _partitions(partitions), _partition(partition) {
+ _idx = use_empty ? _partitions->leftmost_empty(_partition) : _partitions->leftmost(_partition);
+ _end = use_empty ? _partitions->rightmost_empty(_partition) : _partitions->rightmost(_partition);
+ }
+
+ bool has_next() const {
+ if (_idx <= _end) {
+ assert(_partitions->in_free_set(_partition, _idx), "Boundaries or find_last_set_bit failed: " SSIZE_FORMAT, _idx);
+ return true;
+ }
+ return false;
+ }
+
+ idx_t current() const {
+ return _idx;
+ }
+
+ idx_t next() {
+ _idx = _partitions->find_index_of_next_available_region(_partition, _idx + 1);
+ return current();
+ }
+};
+
+class ShenandoahRightLeftIterator {
+private:
+ idx_t _idx;
+ idx_t _end;
+ ShenandoahRegionPartitions* _partitions;
+ ShenandoahFreeSetPartitionId _partition;
+public:
+ explicit ShenandoahRightLeftIterator(ShenandoahRegionPartitions* partitions, ShenandoahFreeSetPartitionId partition, bool use_empty = false)
+ : _idx(0), _end(0), _partitions(partitions), _partition(partition) {
+ _idx = use_empty ? _partitions->rightmost_empty(_partition) : _partitions->rightmost(_partition);
+ _end = use_empty ? _partitions->leftmost_empty(_partition) : _partitions->leftmost(_partition);
+ }
+
+ bool has_next() const {
+ if (_idx >= _end) {
+ assert(_partitions->in_free_set(_partition, _idx), "Boundaries or find_last_set_bit failed: " SSIZE_FORMAT, _idx);
+ return true;
+ }
+ return false;
+ }
+
+ idx_t current() const {
+ return _idx;
+ }
+
+ idx_t next() {
+ _idx = _partitions->find_index_of_previous_available_region(_partition, _idx - 1);
+ return current();
+ }
+};
+
#ifndef PRODUCT
void ShenandoahRegionPartitions::dump_bitmap() const {
- log_debug(gc)("Mutator range [" SSIZE_FORMAT ", " SSIZE_FORMAT "], Collector range [" SSIZE_FORMAT ", " SSIZE_FORMAT "]",
- _leftmosts[int(ShenandoahFreeSetPartitionId::Mutator)],
- _rightmosts[int(ShenandoahFreeSetPartitionId::Mutator)],
- _leftmosts[int(ShenandoahFreeSetPartitionId::Collector)],
- _rightmosts[int(ShenandoahFreeSetPartitionId::Collector)]);
+ log_debug(gc)("Mutator range [" SSIZE_FORMAT ", " SSIZE_FORMAT "], Collector range [" SSIZE_FORMAT ", " SSIZE_FORMAT
+ "], Old Collector range [" SSIZE_FORMAT ", " SSIZE_FORMAT "]",
+ _leftmosts[int(ShenandoahFreeSetPartitionId::Mutator)],
+ _rightmosts[int(ShenandoahFreeSetPartitionId::Mutator)],
+ _leftmosts[int(ShenandoahFreeSetPartitionId::Collector)],
+ _rightmosts[int(ShenandoahFreeSetPartitionId::Collector)],
+ _leftmosts[int(ShenandoahFreeSetPartitionId::OldCollector)],
+ _rightmosts[int(ShenandoahFreeSetPartitionId::OldCollector)]);
log_debug(gc)("Empty Mutator range [" SSIZE_FORMAT ", " SSIZE_FORMAT
- "], Empty Collector range [" SSIZE_FORMAT ", " SSIZE_FORMAT "]",
- _leftmosts_empty[int(ShenandoahFreeSetPartitionId::Mutator)],
- _rightmosts_empty[int(ShenandoahFreeSetPartitionId::Mutator)],
- _leftmosts_empty[int(ShenandoahFreeSetPartitionId::Collector)],
- _rightmosts_empty[int(ShenandoahFreeSetPartitionId::Collector)]);
-
- log_debug(gc)("%6s: %18s %18s %18s", "index", "Mutator Bits", "Collector Bits", "NotFree Bits");
+ "], Empty Collector range [" SSIZE_FORMAT ", " SSIZE_FORMAT
+ "], Empty Old Collecto range [" SSIZE_FORMAT ", " SSIZE_FORMAT "]",
+ _leftmosts_empty[int(ShenandoahFreeSetPartitionId::Mutator)],
+ _rightmosts_empty[int(ShenandoahFreeSetPartitionId::Mutator)],
+ _leftmosts_empty[int(ShenandoahFreeSetPartitionId::Collector)],
+ _rightmosts_empty[int(ShenandoahFreeSetPartitionId::Collector)],
+ _leftmosts_empty[int(ShenandoahFreeSetPartitionId::OldCollector)],
+ _rightmosts_empty[int(ShenandoahFreeSetPartitionId::OldCollector)]);
+
+ log_debug(gc)("%6s: %18s %18s %18s %18s", "index", "Mutator Bits", "Collector Bits", "Old Collector Bits", "NotFree Bits");
dump_bitmap_range(0, _max-1);
}
@@ -81,10 +153,11 @@ void ShenandoahRegionPartitions::dump_bitmap_row(idx_t region_idx) const {
idx_t aligned_idx = _membership[int(ShenandoahFreeSetPartitionId::Mutator)].aligned_index(region_idx);
uintx mutator_bits = _membership[int(ShenandoahFreeSetPartitionId::Mutator)].bits_at(aligned_idx);
uintx collector_bits = _membership[int(ShenandoahFreeSetPartitionId::Collector)].bits_at(aligned_idx);
- uintx free_bits = mutator_bits | collector_bits;
+ uintx old_collector_bits = _membership[int(ShenandoahFreeSetPartitionId::OldCollector)].bits_at(aligned_idx);
+ uintx free_bits = mutator_bits | collector_bits | old_collector_bits;
uintx notfree_bits = ~free_bits;
- log_debug(gc)(SSIZE_FORMAT_W(6) ": " SIZE_FORMAT_X_0 " 0x" SIZE_FORMAT_X_0 " 0x" SIZE_FORMAT_X_0,
- aligned_idx, mutator_bits, collector_bits, notfree_bits);
+ log_debug(gc)(SSIZE_FORMAT_W(6) ": " SIZE_FORMAT_X_0 " 0x" SIZE_FORMAT_X_0 " 0x" SIZE_FORMAT_X_0 " 0x" SIZE_FORMAT_X_0,
+ aligned_idx, mutator_bits, collector_bits, old_collector_bits, notfree_bits);
}
#endif
@@ -92,7 +165,7 @@ ShenandoahRegionPartitions::ShenandoahRegionPartitions(size_t max_regions, Shena
_max(max_regions),
_region_size_bytes(ShenandoahHeapRegion::region_size_bytes()),
_free_set(free_set),
- _membership{ ShenandoahSimpleBitMap(max_regions), ShenandoahSimpleBitMap(max_regions) }
+ _membership{ ShenandoahSimpleBitMap(max_regions), ShenandoahSimpleBitMap(max_regions) , ShenandoahSimpleBitMap(max_regions) }
{
make_all_regions_unavailable();
}
@@ -162,7 +235,6 @@ void ShenandoahRegionPartitions::make_all_regions_unavailable() {
void ShenandoahRegionPartitions::establish_mutator_intervals(idx_t mutator_leftmost, idx_t mutator_rightmost,
idx_t mutator_leftmost_empty, idx_t mutator_rightmost_empty,
size_t mutator_region_count, size_t mutator_used) {
- _region_counts[int(ShenandoahFreeSetPartitionId::Mutator)] = mutator_region_count;
_leftmosts[int(ShenandoahFreeSetPartitionId::Mutator)] = mutator_leftmost;
_rightmosts[int(ShenandoahFreeSetPartitionId::Mutator)] = mutator_rightmost;
_leftmosts_empty[int(ShenandoahFreeSetPartitionId::Mutator)] = mutator_leftmost_empty;
@@ -182,6 +254,20 @@ void ShenandoahRegionPartitions::establish_mutator_intervals(idx_t mutator_leftm
_capacity[int(ShenandoahFreeSetPartitionId::Collector)] = 0;
}
+void ShenandoahRegionPartitions::establish_old_collector_intervals(idx_t old_collector_leftmost, idx_t old_collector_rightmost,
+ idx_t old_collector_leftmost_empty,
+ idx_t old_collector_rightmost_empty,
+ size_t old_collector_region_count, size_t old_collector_used) {
+ _leftmosts[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_leftmost;
+ _rightmosts[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_rightmost;
+ _leftmosts_empty[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_leftmost_empty;
+ _rightmosts_empty[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_rightmost_empty;
+
+ _region_counts[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_region_count;
+ _used[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_used;
+ _capacity[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_region_count * _region_size_bytes;
+}
+
void ShenandoahRegionPartitions::increase_used(ShenandoahFreeSetPartitionId which_partition, size_t bytes) {
assert (which_partition < NumPartitions, "Partition must be valid");
_used[int(which_partition)] += bytes;
@@ -202,7 +288,7 @@ inline void ShenandoahRegionPartitions::shrink_interval_if_range_modifies_either
}
if (_leftmosts_empty[int(partition)] < _leftmosts[int(partition)]) {
// This gets us closer to where we need to be; we'll scan further when leftmosts_empty is requested.
- _leftmosts_empty[int(partition)] = leftmost(partition);
+ _leftmosts_empty[int(partition)] = _leftmosts[int(partition)];
}
}
if (high_idx == _rightmosts[int(partition)]) {
@@ -289,31 +375,61 @@ void ShenandoahRegionPartitions::make_free(idx_t idx, ShenandoahFreeSetPartition
_capacity[int(which_partition)] += _region_size_bytes;
_used[int(which_partition)] += _region_size_bytes - available;
expand_interval_if_boundary_modified(which_partition, idx, available);
-
_region_counts[int(which_partition)]++;
}
+bool ShenandoahRegionPartitions::is_mutator_partition(ShenandoahFreeSetPartitionId p) {
+ return (p == ShenandoahFreeSetPartitionId::Mutator);
+}
+
+bool ShenandoahRegionPartitions::is_young_collector_partition(ShenandoahFreeSetPartitionId p) {
+ return (p == ShenandoahFreeSetPartitionId::Collector);
+}
+
+bool ShenandoahRegionPartitions::is_old_collector_partition(ShenandoahFreeSetPartitionId p) {
+ return (p == ShenandoahFreeSetPartitionId::OldCollector);
+}
+
+bool ShenandoahRegionPartitions::available_implies_empty(size_t available_in_region) {
+ return (available_in_region == _region_size_bytes);
+}
+
+
void ShenandoahRegionPartitions::move_from_partition_to_partition(idx_t idx, ShenandoahFreeSetPartitionId orig_partition,
ShenandoahFreeSetPartitionId new_partition, size_t available) {
+ ShenandoahHeapRegion* r = ShenandoahHeap::heap()->get_region(idx);
assert (idx < _max, "index is sane: " SIZE_FORMAT " < " SIZE_FORMAT, idx, _max);
assert (orig_partition < NumPartitions, "Original partition must be valid");
assert (new_partition < NumPartitions, "New partition must be valid");
assert (available <= _region_size_bytes, "Available cannot exceed region size");
+ assert (_membership[int(orig_partition)].is_set(idx), "Cannot move from partition unless in partition");
+ assert ((r != nullptr) && ((r->is_trash() && (available == _region_size_bytes)) ||
+ (r->used() + available == _region_size_bytes)),
+ "Used: " SIZE_FORMAT " + available: " SIZE_FORMAT " should equal region size: " SIZE_FORMAT,
+ ShenandoahHeap::heap()->get_region(idx)->used(), available, _region_size_bytes);
// Expected transitions:
// During rebuild: Mutator => Collector
+ // Mutator empty => Collector
+ // Mutator empty => OldCollector
// During flip_to_gc: Mutator empty => Collector
+ // Mutator empty => OldCollector
// At start of update refs: Collector => Mutator
- assert (((available <= _region_size_bytes) &&
- (((orig_partition == ShenandoahFreeSetPartitionId::Mutator)
- && (new_partition == ShenandoahFreeSetPartitionId::Collector)) ||
- ((orig_partition == ShenandoahFreeSetPartitionId::Collector)
- && (new_partition == ShenandoahFreeSetPartitionId::Mutator)))) ||
- ((available == _region_size_bytes) &&
- ((orig_partition == ShenandoahFreeSetPartitionId::Mutator)
- && (new_partition == ShenandoahFreeSetPartitionId::Collector))), "Unexpected movement between partitions");
+ // OldCollector Empty => Mutator
+ assert ((is_mutator_partition(orig_partition) && is_young_collector_partition(new_partition)) ||
+ (is_mutator_partition(orig_partition) &&
+ available_implies_empty(available) && is_old_collector_partition(new_partition)) ||
+ (is_young_collector_partition(orig_partition) && is_mutator_partition(new_partition)) ||
+ (is_old_collector_partition(orig_partition)
+ && available_implies_empty(available) && is_mutator_partition(new_partition)),
+ "Unexpected movement between partitions, available: " SIZE_FORMAT ", _region_size_bytes: " SIZE_FORMAT
+ ", orig_partition: %s, new_partition: %s",
+ available, _region_size_bytes, partition_name(orig_partition), partition_name(new_partition));
size_t used = _region_size_bytes - available;
+ assert (_used[int(orig_partition)] >= used,
+ "Orig partition used: " SIZE_FORMAT " must exceed moved used: " SIZE_FORMAT " within region " SSIZE_FORMAT,
+ _used[int(orig_partition)], used, idx);
_membership[int(orig_partition)].clear_bit(idx);
_membership[int(new_partition)].set_bit(idx);
@@ -482,6 +598,7 @@ void ShenandoahRegionPartitions::assert_bounds() {
case ShenandoahFreeSetPartitionId::Mutator:
case ShenandoahFreeSetPartitionId::Collector:
+ case ShenandoahFreeSetPartitionId::OldCollector:
{
size_t capacity = _free_set->alloc_capacity(i);
bool is_empty = (capacity == _region_size_bytes);
@@ -571,6 +688,41 @@ void ShenandoahRegionPartitions::assert_bounds() {
assert (end_off <= _rightmosts_empty[int(ShenandoahFreeSetPartitionId::Collector)],
"free empty regions past the rightmost: " SSIZE_FORMAT ", bound " SSIZE_FORMAT,
end_off, rightmost_empty(ShenandoahFreeSetPartitionId::Collector));
+
+ // Performance invariants. Failing these would not break the free partition, but performance would suffer.
+ assert (leftmost(ShenandoahFreeSetPartitionId::OldCollector) <= _max, "leftmost in bounds: " SSIZE_FORMAT " < " SSIZE_FORMAT,
+ leftmost(ShenandoahFreeSetPartitionId::OldCollector), _max);
+ assert (rightmost(ShenandoahFreeSetPartitionId::OldCollector) < _max, "rightmost in bounds: " SSIZE_FORMAT " < " SSIZE_FORMAT,
+ rightmost(ShenandoahFreeSetPartitionId::OldCollector), _max);
+
+ assert (leftmost(ShenandoahFreeSetPartitionId::OldCollector) == _max
+ || partition_id_matches(leftmost(ShenandoahFreeSetPartitionId::OldCollector),
+ ShenandoahFreeSetPartitionId::OldCollector),
+ "leftmost region should be free: " SSIZE_FORMAT, leftmost(ShenandoahFreeSetPartitionId::OldCollector));
+ assert (leftmost(ShenandoahFreeSetPartitionId::OldCollector) == _max
+ || partition_id_matches(rightmost(ShenandoahFreeSetPartitionId::OldCollector),
+ ShenandoahFreeSetPartitionId::OldCollector),
+ "rightmost region should be free: " SSIZE_FORMAT, rightmost(ShenandoahFreeSetPartitionId::OldCollector));
+
+ // If OldCollector partition is empty, leftmosts will both equal max, rightmosts will both equal zero.
+ // Likewise for empty region partitions.
+ beg_off = leftmosts[int(ShenandoahFreeSetPartitionId::OldCollector)];
+ end_off = rightmosts[int(ShenandoahFreeSetPartitionId::OldCollector)];
+ assert (beg_off >= leftmost(ShenandoahFreeSetPartitionId::OldCollector),
+ "free regions before the leftmost: " SSIZE_FORMAT ", bound " SSIZE_FORMAT,
+ beg_off, leftmost(ShenandoahFreeSetPartitionId::OldCollector));
+ assert (end_off <= rightmost(ShenandoahFreeSetPartitionId::OldCollector),
+ "free regions past the rightmost: " SSIZE_FORMAT ", bound " SSIZE_FORMAT,
+ end_off, rightmost(ShenandoahFreeSetPartitionId::OldCollector));
+
+ beg_off = empty_leftmosts[int(ShenandoahFreeSetPartitionId::OldCollector)];
+ end_off = empty_rightmosts[int(ShenandoahFreeSetPartitionId::OldCollector)];
+ assert (beg_off >= _leftmosts_empty[int(ShenandoahFreeSetPartitionId::OldCollector)],
+ "free empty regions before the leftmost: " SSIZE_FORMAT ", bound " SSIZE_FORMAT,
+ beg_off, leftmost_empty(ShenandoahFreeSetPartitionId::OldCollector));
+ assert (end_off <= _rightmosts_empty[int(ShenandoahFreeSetPartitionId::OldCollector)],
+ "free empty regions past the rightmost: " SSIZE_FORMAT ", bound " SSIZE_FORMAT,
+ end_off, rightmost_empty(ShenandoahFreeSetPartitionId::OldCollector));
}
#endif
@@ -578,150 +730,277 @@ ShenandoahFreeSet::ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions) :
_heap(heap),
_partitions(max_regions, this),
_trash_regions(NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, max_regions, mtGC)),
- _right_to_left_bias(false),
_alloc_bias_weight(0)
{
clear_internal();
}
+void ShenandoahFreeSet::add_promoted_in_place_region_to_old_collector(ShenandoahHeapRegion* region) {
+ shenandoah_assert_heaplocked();
+ size_t plab_min_size_in_bytes = ShenandoahGenerationalHeap::heap()->plab_min_size() * HeapWordSize;
+ size_t idx = region->index();
+ size_t capacity = alloc_capacity(region);
+ assert(_partitions.membership(idx) == ShenandoahFreeSetPartitionId::NotFree,
+ "Regions promoted in place should have been excluded from Mutator partition");
+ if (capacity >= plab_min_size_in_bytes) {
+ _partitions.make_free(idx, ShenandoahFreeSetPartitionId::OldCollector, capacity);
+ _heap->old_generation()->augment_promoted_reserve(capacity);
+ }
+}
+
+HeapWord* ShenandoahFreeSet::allocate_from_partition_with_affiliation(ShenandoahAffiliation affiliation,
+ ShenandoahAllocRequest& req, bool& in_new_region) {
+
+ shenandoah_assert_heaplocked();
+ ShenandoahFreeSetPartitionId which_partition = req.is_old()? ShenandoahFreeSetPartitionId::OldCollector: ShenandoahFreeSetPartitionId::Collector;
+ if (_partitions.alloc_from_left_bias(which_partition)) {
+ ShenandoahLeftRightIterator iterator(&_partitions, which_partition, affiliation == ShenandoahAffiliation::FREE);
+ return allocate_with_affiliation(iterator, affiliation, req, in_new_region);
+ } else {
+ ShenandoahRightLeftIterator iterator(&_partitions, which_partition, affiliation == ShenandoahAffiliation::FREE);
+ return allocate_with_affiliation(iterator, affiliation, req, in_new_region);
+ }
+}
+
+template
+HeapWord* ShenandoahFreeSet::allocate_with_affiliation(Iter& iterator, ShenandoahAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region) {
+ for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) {
+ ShenandoahHeapRegion* r = _heap->get_region(idx);
+ if (r->affiliation() == affiliation) {
+ HeapWord* result = try_allocate_in(r, req, in_new_region);
+ if (result != nullptr) {
+ return result;
+ }
+ }
+ }
+ log_debug(gc, free)("Could not allocate collector region with affiliation: %s for request " PTR_FORMAT,
+ shenandoah_affiliation_name(affiliation), p2i(&req));
+ return nullptr;
+}
+
HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& in_new_region) {
shenandoah_assert_heaplocked();
// Scan the bitmap looking for a first fit.
//
- // Leftmost and rightmost bounds provide enough caching to quickly find a region from which to allocate.
+ // Leftmost and rightmost bounds provide enough caching to walk bitmap efficiently. Normally,
+ // we would find the region to allocate at right away.
//
// Allocations are biased: GC allocations are taken from the high end of the heap. Regular (and TLAB)
// mutator allocations are taken from the middle of heap, below the memory reserved for Collector.
// Humongous mutator allocations are taken from the bottom of the heap.
//
- // Free set maintains mutator and collector partitions. Mutator can only allocate from the
- // Mutator partition. Collector prefers to allocate from the Collector partition, but may steal
- // regions from the Mutator partition if the Collector partition has been depleted.
+ // Free set maintains mutator and collector partitions. Normally, each allocates only from its partition,
+ // except in special cases when the collector steals regions from the mutator partition.
+
+ // Overwrite with non-zero (non-NULL) values only if necessary for allocation bookkeeping.
switch (req.type()) {
case ShenandoahAllocRequest::_alloc_tlab:
- case ShenandoahAllocRequest::_alloc_shared: {
- // Try to allocate in the mutator view
- if (_alloc_bias_weight-- <= 0) {
- // We have observed that regions not collected in previous GC cycle tend to congregate at one end or the other
- // of the heap. Typically, these are the more recently engaged regions and the objects in these regions have not
- // yet had a chance to die (and/or are treated as floating garbage). If we use the same allocation bias on each
- // GC pass, these "most recently" engaged regions for GC pass N will also be the "most recently" engaged regions
- // for GC pass N+1, and the relatively large amount of live data and/or floating garbage introduced
- // during the most recent GC pass may once again prevent the region from being collected. We have found that
- // alternating the allocation behavior between GC passes improves evacuation performance by 3-7% on certain
- // benchmarks. In the best case, this has the effect of consuming these partially consumed regions before
- // the start of the next mark cycle so all of their garbage can be efficiently reclaimed.
- //
- // First, finish consuming regions that are already partially consumed so as to more tightly limit ranges of
- // available regions. Other potential benefits:
- // 1. Eventual collection set has fewer regions because we have packed newly allocated objects into fewer regions
- // 2. We preserve the "empty" regions longer into the GC cycle, reducing likelihood of allocation failures
- // late in the GC cycle.
- idx_t non_empty_on_left = (_partitions.leftmost_empty(ShenandoahFreeSetPartitionId::Mutator)
- - _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator));
- idx_t non_empty_on_right = (_partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator)
- - _partitions.rightmost_empty(ShenandoahFreeSetPartitionId::Mutator));
- _right_to_left_bias = (non_empty_on_right > non_empty_on_left);
- _alloc_bias_weight = _InitialAllocBiasWeight;
- }
- if (_right_to_left_bias) {
- // Allocate within mutator free from high memory to low so as to preserve low memory for humongous allocations
- if (!_partitions.is_empty(ShenandoahFreeSetPartitionId::Mutator)) {
- // Use signed idx. Otherwise, loop will never terminate.
- idx_t leftmost = _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator);
- for (idx_t idx = _partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator); idx >= leftmost; ) {
- assert(_partitions.in_free_set(ShenandoahFreeSetPartitionId::Mutator, idx),
- "Boundaries or find_last_set_bit failed: " SSIZE_FORMAT, idx);
- ShenandoahHeapRegion* r = _heap->get_region(idx);
- // try_allocate_in() increases used if the allocation is successful.
- HeapWord* result;
- size_t min_size = (req.type() == ShenandoahAllocRequest::_alloc_tlab)? req.min_size(): req.size();
- if ((alloc_capacity(r) >= min_size) && ((result = try_allocate_in(r, req, in_new_region)) != nullptr)) {
- return result;
- }
- idx = _partitions.find_index_of_previous_available_region(ShenandoahFreeSetPartitionId::Mutator, idx - 1);
- }
- }
- } else {
- // Allocate from low to high memory. This keeps the range of fully empty regions more tightly packed.
- // Note that the most recently allocated regions tend not to be evacuated in a given GC cycle. So this
- // tends to accumulate "fragmented" uncollected regions in high memory.
- if (!_partitions.is_empty(ShenandoahFreeSetPartitionId::Mutator)) {
- // Use signed idx. Otherwise, loop will never terminate.
- idx_t rightmost = _partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator);
- for (idx_t idx = _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator); idx <= rightmost; ) {
- assert(_partitions.in_free_set(ShenandoahFreeSetPartitionId::Mutator, idx),
- "Boundaries or find_last_set_bit failed: " SSIZE_FORMAT, idx);
- ShenandoahHeapRegion* r = _heap->get_region(idx);
- // try_allocate_in() increases used if the allocation is successful.
- HeapWord* result;
- size_t min_size = (req.type() == ShenandoahAllocRequest::_alloc_tlab)? req.min_size(): req.size();
- if ((alloc_capacity(r) >= min_size) && ((result = try_allocate_in(r, req, in_new_region)) != nullptr)) {
- return result;
- }
- idx = _partitions.find_index_of_next_available_region(ShenandoahFreeSetPartitionId::Mutator, idx + 1);
- }
- }
- }
- // There is no recovery. Mutator does not touch collector view at all.
- break;
- }
+ case ShenandoahAllocRequest::_alloc_shared:
+ return allocate_for_mutator(req, in_new_region);
case ShenandoahAllocRequest::_alloc_gclab:
- // GCLABs are for evacuation so we must be in evacuation phase.
-
- case ShenandoahAllocRequest::_alloc_shared_gc: {
- // Fast-path: try to allocate in the collector view first
- idx_t leftmost_collector = _partitions.leftmost(ShenandoahFreeSetPartitionId::Collector);
- for (idx_t idx = _partitions.rightmost(ShenandoahFreeSetPartitionId::Collector); idx >= leftmost_collector; ) {
- assert(_partitions.in_free_set(ShenandoahFreeSetPartitionId::Collector, idx),
- "Boundaries or find_prev_last_bit failed: " SSIZE_FORMAT, idx);
- HeapWord* result = try_allocate_in(_heap->get_region(idx), req, in_new_region);
- if (result != nullptr) {
- return result;
- }
- idx = _partitions.find_index_of_previous_available_region(ShenandoahFreeSetPartitionId::Collector, idx - 1);
- }
+ case ShenandoahAllocRequest::_alloc_plab:
+ case ShenandoahAllocRequest::_alloc_shared_gc:
+ return allocate_for_collector(req, in_new_region);
+ default:
+ ShouldNotReachHere();
+ }
+ return nullptr;
+}
- // No dice. Can we borrow space from mutator view?
- if (!ShenandoahEvacReserveOverflow) {
- return nullptr;
- }
+HeapWord* ShenandoahFreeSet::allocate_for_mutator(ShenandoahAllocRequest &req, bool &in_new_region) {
+ update_allocation_bias();
- // Try to steal an empty region from the mutator view.
- idx_t leftmost_mutator_empty = _partitions.leftmost_empty(ShenandoahFreeSetPartitionId::Mutator);
- for (idx_t idx = _partitions.rightmost_empty(ShenandoahFreeSetPartitionId::Mutator); idx >= leftmost_mutator_empty; ) {
- assert(_partitions.in_free_set(ShenandoahFreeSetPartitionId::Mutator, idx),
- "Boundaries or find_prev_last_bit failed: " SSIZE_FORMAT, idx);
- ShenandoahHeapRegion* r = _heap->get_region(idx);
- if (can_allocate_from(r)) {
- flip_to_gc(r);
- HeapWord *result = try_allocate_in(r, req, in_new_region);
- if (result != nullptr) {
- log_debug(gc)("Flipped region " SIZE_FORMAT " to gc for request: " PTR_FORMAT, idx, p2i(&req));
- return result;
- }
- }
- idx = _partitions.find_index_of_previous_available_region(ShenandoahFreeSetPartitionId::Mutator, idx - 1);
+ if (_partitions.is_empty(ShenandoahFreeSetPartitionId::Mutator)) {
+ // There is no recovery. Mutator does not touch collector view at all.
+ return nullptr;
+ }
+
+ // Try to allocate in the mutator view
+ if (_partitions.alloc_from_left_bias(ShenandoahFreeSetPartitionId::Mutator)) {
+ // Allocate from low to high memory. This keeps the range of fully empty regions more tightly packed.
+ // Note that the most recently allocated regions tend not to be evacuated in a given GC cycle. So this
+ // tends to accumulate "fragmented" uncollected regions in high memory.
+ ShenandoahLeftRightIterator iterator(&_partitions, ShenandoahFreeSetPartitionId::Mutator);
+ return allocate_from_regions(iterator, req, in_new_region);
+ }
+
+ // Allocate from high to low memory. This preserves low memory for humongous allocations.
+ ShenandoahRightLeftIterator iterator(&_partitions, ShenandoahFreeSetPartitionId::Mutator);
+ return allocate_from_regions(iterator, req, in_new_region);
+}
+
+void ShenandoahFreeSet::update_allocation_bias() {
+ if (_alloc_bias_weight-- <= 0) {
+ // We have observed that regions not collected in previous GC cycle tend to congregate at one end or the other
+ // of the heap. Typically, these are the more recently engaged regions and the objects in these regions have not
+ // yet had a chance to die (and/or are treated as floating garbage). If we use the same allocation bias on each
+ // GC pass, these "most recently" engaged regions for GC pass N will also be the "most recently" engaged regions
+ // for GC pass N+1, and the relatively large amount of live data and/or floating garbage introduced
+ // during the most recent GC pass may once again prevent the region from being collected. We have found that
+ // alternating the allocation behavior between GC passes improves evacuation performance by 3-7% on certain
+ // benchmarks. In the best case, this has the effect of consuming these partially consumed regions before
+ // the start of the next mark cycle so all of their garbage can be efficiently reclaimed.
+ //
+ // First, finish consuming regions that are already partially consumed so as to more tightly limit ranges of
+ // available regions. Other potential benefits:
+ // 1. Eventual collection set has fewer regions because we have packed newly allocated objects into fewer regions
+ // 2. We preserve the "empty" regions longer into the GC cycle, reducing likelihood of allocation failures
+ // late in the GC cycle.
+ idx_t non_empty_on_left = (_partitions.leftmost_empty(ShenandoahFreeSetPartitionId::Mutator)
+ - _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator));
+ idx_t non_empty_on_right = (_partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator)
+ - _partitions.rightmost_empty(ShenandoahFreeSetPartitionId::Mutator));
+ _partitions.set_bias_from_left_to_right(ShenandoahFreeSetPartitionId::Mutator, (non_empty_on_right < non_empty_on_left));
+ _alloc_bias_weight = INITIAL_ALLOC_BIAS_WEIGHT;
+ }
+}
+
+template
+HeapWord* ShenandoahFreeSet::allocate_from_regions(Iter& iterator, ShenandoahAllocRequest &req, bool &in_new_region) {
+ for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) {
+ ShenandoahHeapRegion* r = _heap->get_region(idx);
+ size_t min_size = (req.type() == ShenandoahAllocRequest::_alloc_tlab) ? req.min_size() : req.size();
+ if (alloc_capacity(r) >= min_size) {
+ HeapWord* result = try_allocate_in(r, req, in_new_region);
+ if (result != nullptr) {
+ return result;
}
+ }
+ }
+ return nullptr;
+}
- // No dice. Do not try to mix mutator and GC allocations, because adjusting region UWM
- // due to GC allocations would expose unparsable mutator allocations.
- break;
+HeapWord* ShenandoahFreeSet::allocate_for_collector(ShenandoahAllocRequest &req, bool &in_new_region) {
+ // Fast-path: try to allocate in the collector view first
+ HeapWord* result;
+ result = allocate_from_partition_with_affiliation(req.affiliation(), req, in_new_region);
+ if (result != nullptr) {
+ return result;
+ }
+
+ bool allow_new_region = can_allocate_in_new_region(req);
+ if (allow_new_region) {
+ // Try a free region that is dedicated to GC allocations.
+ result = allocate_from_partition_with_affiliation(ShenandoahAffiliation::FREE, req, in_new_region);
+ if (result != nullptr) {
+ return result;
+ }
+ }
+
+ // No dice. Can we borrow space from mutator view?
+ if (!ShenandoahEvacReserveOverflow) {
+ return nullptr;
+ }
+
+ if (!allow_new_region && req.is_old() && (_heap->young_generation()->free_unaffiliated_regions() > 0)) {
+ // This allows us to flip a mutator region to old_collector
+ allow_new_region = true;
+ }
+
+ // We should expand old-gen if this can prevent an old-gen evacuation failure. We don't care so much about
+ // promotion failures since they can be mitigated in a subsequent GC pass. Would be nice to know if this
+ // allocation request is for evacuation or promotion. Individual threads limit their use of PLAB memory for
+ // promotions, so we already have an assurance that any additional memory set aside for old-gen will be used
+ // only for old-gen evacuations.
+ if (allow_new_region) {
+ // Try to steal an empty region from the mutator view.
+ result = try_allocate_from_mutator(req, in_new_region);
+ }
+
+ // This is it. Do not try to mix mutator and GC allocations, because adjusting region UWM
+ // due to GC allocations would expose unparsable mutator allocations.
+ return result;
+}
+
+bool ShenandoahFreeSet::can_allocate_in_new_region(const ShenandoahAllocRequest& req) {
+ if (!_heap->mode()->is_generational()) {
+ return true;
+ }
+
+ assert(req.is_old() || req.is_young(), "Should request affiliation");
+ return (req.is_old() && _heap->old_generation()->free_unaffiliated_regions() > 0)
+ || (req.is_young() && _heap->young_generation()->free_unaffiliated_regions() > 0);
+}
+
+HeapWord* ShenandoahFreeSet::try_allocate_from_mutator(ShenandoahAllocRequest& req, bool& in_new_region) {
+ // The collector prefers to keep longer lived regions toward the right side of the heap, so it always
+ // searches for regions from right to left here.
+ ShenandoahRightLeftIterator iterator(&_partitions, ShenandoahFreeSetPartitionId::Mutator, true);
+ for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) {
+ ShenandoahHeapRegion* r = _heap->get_region(idx);
+ if (can_allocate_from(r)) {
+ if (req.is_old()) {
+ flip_to_old_gc(r);
+ } else {
+ flip_to_gc(r);
+ }
+ // Region r is entirely empty. If try_allocate_in fails on region r, something else is really wrong.
+ // Don't bother to retry with other regions.
+ log_debug(gc, free)("Flipped region " SIZE_FORMAT " to gc for request: " PTR_FORMAT, idx, p2i(&req));
+ return try_allocate_in(r, req, in_new_region);
}
- default:
- ShouldNotReachHere();
}
+
return nullptr;
}
+// This work method takes an argument corresponding to the number of bytes
+// free in a region, and returns the largest amount in heapwords that can be allocated
+// such that both of the following conditions are satisfied:
+//
+// 1. it is a multiple of card size
+// 2. any remaining shard may be filled with a filler object
+//
+// The idea is that the allocation starts and ends at card boundaries. Because
+// a region ('s end) is card-aligned, the remainder shard that must be filled is
+// at the start of the free space.
+//
+// This is merely a helper method to use for the purpose of such a calculation.
+size_t ShenandoahFreeSet::get_usable_free_words(size_t free_bytes) const {
+ // e.g. card_size is 512, card_shift is 9, min_fill_size() is 8
+ // free is 514
+ // usable_free is 512, which is decreased to 0
+ size_t usable_free = (free_bytes / CardTable::card_size()) << CardTable::card_shift();
+ assert(usable_free <= free_bytes, "Sanity check");
+ if ((free_bytes != usable_free) && (free_bytes - usable_free < ShenandoahHeap::min_fill_size() * HeapWordSize)) {
+ // After aligning to card multiples, the remainder would be smaller than
+ // the minimum filler object, so we'll need to take away another card's
+ // worth to construct a filler object.
+ if (usable_free >= CardTable::card_size()) {
+ usable_free -= CardTable::card_size();
+ } else {
+ assert(usable_free == 0, "usable_free is a multiple of card_size and card_size > min_fill_size");
+ }
+ }
+
+ return usable_free / HeapWordSize;
+}
+
+// Given a size argument, which is a multiple of card size, a request struct
+// for a PLAB, and an old region, return a pointer to the allocated space for
+// a PLAB which is card-aligned and where any remaining shard in the region
+// has been suitably filled by a filler object.
+// It is assumed (and assertion-checked) that such an allocation is always possible.
+HeapWord* ShenandoahFreeSet::allocate_aligned_plab(size_t size, ShenandoahAllocRequest& req, ShenandoahHeapRegion* r) {
+ assert(_heap->mode()->is_generational(), "PLABs are only for generational mode");
+ assert(r->is_old(), "All PLABs reside in old-gen");
+ assert(!req.is_mutator_alloc(), "PLABs should not be allocated by mutators.");
+ assert(is_aligned(size, CardTable::card_size_in_words()), "Align by design");
+
+ HeapWord* result = r->allocate_aligned(size, req, CardTable::card_size());
+ assert(result != nullptr, "Allocation cannot fail");
+ assert(r->top() <= r->end(), "Allocation cannot span end of region");
+ assert(is_aligned(result, CardTable::card_size_in_words()), "Align by design");
+ return result;
+}
+
HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, ShenandoahAllocRequest& req, bool& in_new_region) {
assert (has_alloc_capacity(r), "Performance: should avoid full regions on this path: " SIZE_FORMAT, r->index());
if (_heap->is_concurrent_weak_root_in_progress() && r->is_trash()) {
return nullptr;
}
-
HeapWord* result = nullptr;
try_recycle_trashed(r);
in_new_region = r->is_empty();
@@ -729,37 +1008,83 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah
if (in_new_region) {
log_debug(gc)("Using new region (" SIZE_FORMAT ") for %s (" PTR_FORMAT ").",
r->index(), ShenandoahAllocRequest::alloc_type_to_string(req.type()), p2i(&req));
+ assert(!r->is_affiliated(), "New region " SIZE_FORMAT " should be unaffiliated", r->index());
+ r->set_affiliation(req.affiliation());
+ if (r->is_old()) {
+ // Any OLD region allocated during concurrent coalesce-and-fill does not need to be coalesced and filled because
+ // all objects allocated within this region are above TAMS (and thus are implicitly marked). In case this is an
+ // OLD region and concurrent preparation for mixed evacuations visits this region before the start of the next
+ // old-gen concurrent mark (i.e. this region is allocated following the start of old-gen concurrent mark but before
+ // concurrent preparations for mixed evacuations are completed), we mark this region as not requiring any
+ // coalesce-and-fill processing.
+ r->end_preemptible_coalesce_and_fill();
+ _heap->old_generation()->clear_cards_for(r);
+ }
+ _heap->generation_for(r->affiliation())->increment_affiliated_region_count();
+
+#ifdef ASSERT
+ ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
+ assert(ctx->top_at_mark_start(r) == r->bottom(), "Newly established allocation region starts with TAMS equal to bottom");
+ assert(ctx->is_bitmap_range_within_region_clear(ctx->top_bitmap(r), r->end()), "Bitmap above top_bitmap() must be clear");
+#endif
+ log_debug(gc)("Using new region (" SIZE_FORMAT ") for %s (" PTR_FORMAT ").",
+ r->index(), ShenandoahAllocRequest::alloc_type_to_string(req.type()), p2i(&req));
+ } else {
+ assert(r->is_affiliated(), "Region " SIZE_FORMAT " that is not new should be affiliated", r->index());
+ if (r->affiliation() != req.affiliation()) {
+ assert(_heap->mode()->is_generational(), "Request for %s from %s region should only happen in generational mode.",
+ req.affiliation_name(), r->affiliation_name());
+ return nullptr;
+ }
}
// req.size() is in words, r->free() is in bytes.
if (req.is_lab_alloc()) {
- // This is a GCLAB or a TLAB allocation
size_t adjusted_size = req.size();
- size_t free = align_down(r->free() >> LogHeapWordSize, MinObjAlignment);
- if (adjusted_size > free) {
- adjusted_size = free;
- }
- if (adjusted_size >= req.min_size()) {
- result = r->allocate(adjusted_size, req.type());
- log_debug(gc)("Allocated " SIZE_FORMAT " words (adjusted from " SIZE_FORMAT ") for %s @" PTR_FORMAT
- " from %s region " SIZE_FORMAT ", free bytes remaining: " SIZE_FORMAT,
- adjusted_size, req.size(), ShenandoahAllocRequest::alloc_type_to_string(req.type()), p2i(result),
- _partitions.partition_membership_name(r->index()), r->index(), r->free());
- assert (result != nullptr, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, free, adjusted_size);
- req.set_actual_size(adjusted_size);
+ size_t free = r->free(); // free represents bytes available within region r
+ if (req.type() == ShenandoahAllocRequest::_alloc_plab) {
+ // This is a PLAB allocation
+ assert(_heap->mode()->is_generational(), "PLABs are only for generational mode");
+ assert(_partitions.in_free_set(ShenandoahFreeSetPartitionId::OldCollector, r->index()),
+ "PLABS must be allocated in old_collector_free regions");
+
+ // Need to assure that plabs are aligned on multiple of card region
+ // Convert free from unaligned bytes to aligned number of words
+ size_t usable_free = get_usable_free_words(free);
+ if (adjusted_size > usable_free) {
+ adjusted_size = usable_free;
+ }
+ adjusted_size = align_down(adjusted_size, CardTable::card_size_in_words());
+ if (adjusted_size >= req.min_size()) {
+ result = allocate_aligned_plab(adjusted_size, req, r);
+ assert(result != nullptr, "allocate must succeed");
+ req.set_actual_size(adjusted_size);
+ } else {
+ // Otherwise, leave result == nullptr because the adjusted size is smaller than min size.
+ log_trace(gc, free)("Failed to shrink PLAB request (" SIZE_FORMAT ") in region " SIZE_FORMAT " to " SIZE_FORMAT
+ " because min_size() is " SIZE_FORMAT, req.size(), r->index(), adjusted_size, req.min_size());
+ }
} else {
- log_trace(gc, free)("Failed to shrink TLAB or GCLAB request (" SIZE_FORMAT ") in region " SIZE_FORMAT " to " SIZE_FORMAT
- " because min_size() is " SIZE_FORMAT, req.size(), r->index(), adjusted_size, req.min_size());
+ // This is a GCLAB or a TLAB allocation
+ // Convert free from unaligned bytes to aligned number of words
+ free = align_down(free >> LogHeapWordSize, MinObjAlignment);
+ if (adjusted_size > free) {
+ adjusted_size = free;
+ }
+ if (adjusted_size >= req.min_size()) {
+ result = r->allocate(adjusted_size, req);
+ assert (result != nullptr, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, free, adjusted_size);
+ req.set_actual_size(adjusted_size);
+ } else {
+ log_trace(gc, free)("Failed to shrink TLAB or GCLAB request (" SIZE_FORMAT ") in region " SIZE_FORMAT " to " SIZE_FORMAT
+ " because min_size() is " SIZE_FORMAT, req.size(), r->index(), adjusted_size, req.min_size());
+ }
}
} else {
size_t size = req.size();
- result = r->allocate(size, req.type());
+ result = r->allocate(size, req);
if (result != nullptr) {
// Record actual allocation size
- log_debug(gc)("Allocated " SIZE_FORMAT " words for %s @" PTR_FORMAT
- " from %s region " SIZE_FORMAT ", free bytes remaining: " SIZE_FORMAT,
- size, ShenandoahAllocRequest::alloc_type_to_string(req.type()), p2i(result),
- _partitions.partition_membership_name(r->index()), r->index(), r->free());
req.set_actual_size(size);
}
}
@@ -767,13 +1092,22 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah
if (result != nullptr) {
// Allocation successful, bump stats:
if (req.is_mutator_alloc()) {
+ assert(req.is_young(), "Mutator allocations always come from young generation.");
_partitions.increase_used(ShenandoahFreeSetPartitionId::Mutator, req.actual_size() * HeapWordSize);
} else {
assert(req.is_gc_alloc(), "Should be gc_alloc since req wasn't mutator alloc");
// For GC allocations, we advance update_watermark because the objects relocated into this memory during
- // evacuation are not updated during evacuation.
+ // evacuation are not updated during evacuation. For both young and old regions r, it is essential that all
+ // PLABs be made parsable at the end of evacuation. This is enabled by retiring all plabs at end of evacuation.
r->set_update_watermark(r->top());
+ if (r->is_old()) {
+ _partitions.increase_used(ShenandoahFreeSetPartitionId::OldCollector, req.actual_size() * HeapWordSize);
+ assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "old-gen allocations use PLAB or shared allocation");
+ // for plabs, we'll sort the difference between evac and promotion usage when we retire the plab
+ } else {
+ _partitions.increase_used(ShenandoahFreeSetPartitionId::Collector, req.actual_size() * HeapWordSize);
+ }
}
}
@@ -788,9 +1122,22 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah
// then retire the region so that subsequent searches can find available memory more quickly.
size_t idx = r->index();
- _partitions.retire_from_partition(req.is_mutator_alloc()?
- ShenandoahFreeSetPartitionId::Mutator: ShenandoahFreeSetPartitionId::Collector,
- idx, r->used());
+ ShenandoahFreeSetPartitionId orig_partition;
+ if (req.is_mutator_alloc()) {
+ orig_partition = ShenandoahFreeSetPartitionId::Mutator;
+ } else if (req.type() == ShenandoahAllocRequest::_alloc_gclab) {
+ orig_partition = ShenandoahFreeSetPartitionId::Collector;
+ } else if (req.type() == ShenandoahAllocRequest::_alloc_plab) {
+ orig_partition = ShenandoahFreeSetPartitionId::OldCollector;
+ } else {
+ assert(req.type() == ShenandoahAllocRequest::_alloc_shared_gc, "Unexpected allocation type");
+ if (req.is_old()) {
+ orig_partition = ShenandoahFreeSetPartitionId::OldCollector;
+ } else {
+ orig_partition = ShenandoahFreeSetPartitionId::Collector;
+ }
+ }
+ _partitions.retire_from_partition(orig_partition, idx, r->used());
_partitions.assert_bounds();
}
return result;
@@ -803,6 +1150,9 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) {
size_t words_size = req.size();
idx_t num = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
+ assert(req.is_young(), "Humongous regions always allocated in YOUNG");
+ ShenandoahGeneration* generation = _heap->generation_for(req.affiliation());
+
// Check if there are enough regions left to satisfy allocation.
if (num > (idx_t) _partitions.count(ShenandoahFreeSetPartitionId::Mutator)) {
return nullptr;
@@ -815,7 +1165,7 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) {
// Find the continuous interval of $num regions, starting from $beg and ending in $end,
// inclusive. Contiguous allocations are biased to the beginning.
idx_t beg = _partitions.find_index_of_next_available_cluster_of_regions(ShenandoahFreeSetPartitionId::Mutator,
- start_range, num);
+ start_range, num);
if (beg > last_possible_start) {
// Hit the end, goodbye
return nullptr;
@@ -882,9 +1232,11 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) {
used_words = ShenandoahHeapRegion::region_size_words();
}
+ r->set_affiliation(req.affiliation());
+ r->set_update_watermark(r->bottom());
r->set_top(r->bottom() + used_words);
}
-
+ generation->increase_affiliated_region_count(num);
if (remainder != 0) {
// Record this remainder as allocation waste
_heap->notify_mutator_alloc_words(ShenandoahHeapRegion::region_size_words() - remainder, true);
@@ -897,12 +1249,14 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) {
_partitions.increase_used(ShenandoahFreeSetPartitionId::Mutator, total_humongous_size);
_partitions.assert_bounds();
req.set_actual_size(words_size);
+ if (remainder != 0) {
+ req.set_waste(ShenandoahHeapRegion::region_size_words() - remainder);
+ }
return _heap->get_region(beg)->bottom();
}
void ShenandoahFreeSet::try_recycle_trashed(ShenandoahHeapRegion* r) {
if (r->is_trash()) {
- _heap->decrease_used(r->used());
r->recycle();
}
}
@@ -960,6 +1314,27 @@ void ShenandoahFreeSet::recycle_trash() {
}
}
+void ShenandoahFreeSet::flip_to_old_gc(ShenandoahHeapRegion* r) {
+ size_t idx = r->index();
+
+ assert(_partitions.partition_id_matches(idx, ShenandoahFreeSetPartitionId::Mutator), "Should be in mutator view");
+ assert(can_allocate_from(r), "Should not be allocated");
+
+ ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap();
+ size_t region_capacity = alloc_capacity(r);
+ _partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Mutator,
+ ShenandoahFreeSetPartitionId::OldCollector, region_capacity);
+ _partitions.assert_bounds();
+ _heap->old_generation()->augment_evacuation_reserve(region_capacity);
+ bool transferred = gen_heap->generation_sizer()->transfer_to_old(1);
+ if (!transferred) {
+ log_warning(gc, free)("Forcing transfer of " SIZE_FORMAT " to old reserve.", idx);
+ gen_heap->generation_sizer()->force_transfer_to_old(1);
+ }
+ // We do not ensure that the region is no longer trash, relying on try_allocate_in(), which always comes next,
+ // to recycle trash before attempting to allocate anything in the region.
+}
+
void ShenandoahFreeSet::flip_to_gc(ShenandoahHeapRegion* r) {
size_t idx = r->index();
@@ -982,12 +1357,24 @@ void ShenandoahFreeSet::clear() {
void ShenandoahFreeSet::clear_internal() {
_partitions.make_all_regions_unavailable();
-}
-void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &cset_regions) {
+ _alloc_bias_weight = 0;
+ _partitions.set_bias_from_left_to_right(ShenandoahFreeSetPartitionId::Mutator, true);
+ _partitions.set_bias_from_left_to_right(ShenandoahFreeSetPartitionId::Collector, false);
+ _partitions.set_bias_from_left_to_right(ShenandoahFreeSetPartitionId::OldCollector, false);
+}
- cset_regions = 0;
+void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regions, size_t &old_cset_regions,
+ size_t &first_old_region, size_t &last_old_region,
+ size_t &old_region_count) {
clear_internal();
+
+ first_old_region = _heap->num_regions();
+ last_old_region = 0;
+ old_region_count = 0;
+ old_cset_regions = 0;
+ young_cset_regions = 0;
+
size_t region_size_bytes = _partitions.region_size_bytes();
size_t max_regions = _partitions.max_regions();
@@ -995,77 +1382,181 @@ void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &cset_regions) {
size_t mutator_rightmost = 0;
size_t mutator_leftmost_empty = max_regions;
size_t mutator_rightmost_empty = 0;
-
size_t mutator_regions = 0;
size_t mutator_used = 0;
- for (size_t idx = 0; idx < _heap->num_regions(); idx++) {
+ size_t old_collector_leftmost = max_regions;
+ size_t old_collector_rightmost = 0;
+ size_t old_collector_leftmost_empty = max_regions;
+ size_t old_collector_rightmost_empty = 0;
+ size_t old_collector_regions = 0;
+ size_t old_collector_used = 0;
+
+ size_t num_regions = _heap->num_regions();
+ for (size_t idx = 0; idx < num_regions; idx++) {
ShenandoahHeapRegion* region = _heap->get_region(idx);
if (region->is_trash()) {
// Trashed regions represent regions that had been in the collection partition but have not yet been "cleaned up".
// The cset regions are not "trashed" until we have finished update refs.
- cset_regions++;
+ if (region->is_old()) {
+ old_cset_regions++;
+ } else {
+ assert(region->is_young(), "Trashed region should be old or young");
+ young_cset_regions++;
+ }
+ } else if (region->is_old()) {
+ // count both humongous and regular regions, but don't count trash (cset) regions.
+ old_region_count++;
+ if (first_old_region > idx) {
+ first_old_region = idx;
+ }
+ last_old_region = idx;
}
if (region->is_alloc_allowed() || region->is_trash()) {
+ assert(!region->is_cset(), "Shouldn't be adding cset regions to the free set");
// Do not add regions that would almost surely fail allocation
size_t ac = alloc_capacity(region);
if (ac > PLAB::min_size() * HeapWordSize) {
- _partitions.raw_assign_membership(idx, ShenandoahFreeSetPartitionId::Mutator);
-
- if (idx < mutator_leftmost) {
- mutator_leftmost = idx;
- }
- if (idx > mutator_rightmost) {
- mutator_rightmost = idx;
- }
- if (ac == region_size_bytes) {
- if (idx < mutator_leftmost_empty) {
- mutator_leftmost_empty = idx;
+ if (region->is_trash() || !region->is_old()) {
+ // Both young and old collected regions (trashed) are placed into the Mutator set
+ _partitions.raw_assign_membership(idx, ShenandoahFreeSetPartitionId::Mutator);
+ if (idx < mutator_leftmost) {
+ mutator_leftmost = idx;
+ }
+ if (idx > mutator_rightmost) {
+ mutator_rightmost = idx;
+ }
+ if (ac == region_size_bytes) {
+ if (idx < mutator_leftmost_empty) {
+ mutator_leftmost_empty = idx;
+ }
+ if (idx > mutator_rightmost_empty) {
+ mutator_rightmost_empty = idx;
+ }
+ }
+ mutator_regions++;
+ mutator_used += (region_size_bytes - ac);
+ } else {
+ // !region->is_trash() && region is_old()
+ _partitions.raw_assign_membership(idx, ShenandoahFreeSetPartitionId::OldCollector);
+ if (idx < old_collector_leftmost) {
+ old_collector_leftmost = idx;
}
- if (idx > mutator_rightmost_empty) {
- mutator_rightmost_empty = idx;
+ if (idx > old_collector_rightmost) {
+ old_collector_rightmost = idx;
}
+ if (ac == region_size_bytes) {
+ if (idx < old_collector_leftmost_empty) {
+ old_collector_leftmost_empty = idx;
+ }
+ if (idx > old_collector_rightmost_empty) {
+ old_collector_rightmost_empty = idx;
+ }
+ }
+ old_collector_regions++;
+ old_collector_used += (region_size_bytes - ac);
}
- mutator_regions++;
- mutator_used += (region_size_bytes - ac);
-
- log_debug(gc)(
- " Adding Region " SIZE_FORMAT " (Free: " SIZE_FORMAT "%s, Used: " SIZE_FORMAT "%s) to mutator partition",
- idx, byte_size_in_proper_unit(region->free()), proper_unit_for_byte_size(region->free()),
- byte_size_in_proper_unit(region->used()), proper_unit_for_byte_size(region->used()));
}
}
}
+ log_debug(gc)(" At end of prep_to_rebuild, mutator_leftmost: " SIZE_FORMAT
+ ", mutator_rightmost: " SIZE_FORMAT
+ ", mutator_leftmost_empty: " SIZE_FORMAT
+ ", mutator_rightmost_empty: " SIZE_FORMAT
+ ", mutator_regions: " SIZE_FORMAT
+ ", mutator_used: " SIZE_FORMAT,
+ mutator_leftmost, mutator_rightmost, mutator_leftmost_empty, mutator_rightmost_empty,
+ mutator_regions, mutator_used);
+
+ log_debug(gc)(" old_collector_leftmost: " SIZE_FORMAT
+ ", old_collector_rightmost: " SIZE_FORMAT
+ ", old_collector_leftmost_empty: " SIZE_FORMAT
+ ", old_collector_rightmost_empty: " SIZE_FORMAT
+ ", old_collector_regions: " SIZE_FORMAT
+ ", old_collector_used: " SIZE_FORMAT,
+ old_collector_leftmost, old_collector_rightmost, old_collector_leftmost_empty, old_collector_rightmost_empty,
+ old_collector_regions, old_collector_used);
+
idx_t rightmost_idx = (mutator_leftmost == max_regions)? -1: (idx_t) mutator_rightmost;
idx_t rightmost_empty_idx = (mutator_leftmost_empty == max_regions)? -1: (idx_t) mutator_rightmost_empty;
_partitions.establish_mutator_intervals(mutator_leftmost, rightmost_idx, mutator_leftmost_empty, rightmost_empty_idx,
mutator_regions, mutator_used);
+ rightmost_idx = (old_collector_leftmost == max_regions)? -1: (idx_t) old_collector_rightmost;
+ rightmost_empty_idx = (old_collector_leftmost_empty == max_regions)? -1: (idx_t) old_collector_rightmost_empty;
+ _partitions.establish_old_collector_intervals(old_collector_leftmost, rightmost_idx, old_collector_leftmost_empty,
+ rightmost_empty_idx, old_collector_regions, old_collector_used);
+ log_debug(gc)(" After find_regions_with_alloc_capacity(), Mutator range [" SSIZE_FORMAT ", " SSIZE_FORMAT "],"
+ " Old Collector range [" SSIZE_FORMAT ", " SSIZE_FORMAT "]",
+ _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator),
+ _partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator),
+ _partitions.leftmost(ShenandoahFreeSetPartitionId::OldCollector),
+ _partitions.rightmost(ShenandoahFreeSetPartitionId::OldCollector));
+}
+
+// Returns number of regions transferred, adds transferred bytes to var argument bytes_transferred
+size_t ShenandoahFreeSet::transfer_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId which_collector,
+ size_t max_xfer_regions,
+ size_t& bytes_transferred) {
+ shenandoah_assert_heaplocked();
+ const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
+ size_t transferred_regions = 0;
+ ShenandoahLeftRightIterator iterator(&_partitions, which_collector, true);
+ idx_t rightmost = _partitions.rightmost_empty(which_collector);
+ for (idx_t idx = iterator.current(); transferred_regions < max_xfer_regions && iterator.has_next(); idx = iterator.next()) {
+ // Note: can_allocate_from() denotes that region is entirely empty
+ if (can_allocate_from(idx)) {
+ _partitions.move_from_partition_to_partition(idx, which_collector, ShenandoahFreeSetPartitionId::Mutator, region_size_bytes);
+ transferred_regions++;
+ bytes_transferred += region_size_bytes;
+ }
+ }
+ return transferred_regions;
+}
+
+// Returns number of regions transferred, adds transferred bytes to var argument bytes_transferred
+size_t ShenandoahFreeSet::transfer_non_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId which_collector,
+ size_t max_xfer_regions,
+ size_t& bytes_transferred) {
+ shenandoah_assert_heaplocked();
+ size_t transferred_regions = 0;
+ ShenandoahLeftRightIterator iterator(&_partitions, which_collector, false);
+ for (idx_t idx = iterator.current(); transferred_regions < max_xfer_regions && iterator.has_next(); idx = iterator.next()) {
+ size_t ac = alloc_capacity(idx);
+ if (ac > 0) {
+ _partitions.move_from_partition_to_partition(idx, which_collector, ShenandoahFreeSetPartitionId::Mutator, ac);
+ transferred_regions++;
+ bytes_transferred += ac;
+ }
+ }
+ return transferred_regions;
}
void ShenandoahFreeSet::move_regions_from_collector_to_mutator(size_t max_xfer_regions) {
- size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
- size_t collector_empty_xfer = 0;
- size_t collector_not_empty_xfer = 0;
+ size_t collector_xfer = 0;
+ size_t old_collector_xfer = 0;
// Process empty regions within the Collector free partition
if ((max_xfer_regions > 0) &&
(_partitions.leftmost_empty(ShenandoahFreeSetPartitionId::Collector)
<= _partitions.rightmost_empty(ShenandoahFreeSetPartitionId::Collector))) {
ShenandoahHeapLocker locker(_heap->lock());
- idx_t rightmost = _partitions.rightmost_empty(ShenandoahFreeSetPartitionId::Collector);
- for (idx_t idx = _partitions.leftmost_empty(ShenandoahFreeSetPartitionId::Collector);
- (max_xfer_regions > 0) && (idx <= rightmost); ) {
- assert(_partitions.in_free_set(ShenandoahFreeSetPartitionId::Collector, idx),
- "Boundaries or find_first_set_bit failed: " SSIZE_FORMAT, idx);
- // Note: can_allocate_from() denotes that region is entirely empty
- if (can_allocate_from(idx)) {
- _partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Collector,
- ShenandoahFreeSetPartitionId::Mutator, region_size_bytes);
- max_xfer_regions--;
- collector_empty_xfer += region_size_bytes;
- }
- idx = _partitions.find_index_of_next_available_region(ShenandoahFreeSetPartitionId::Collector, idx + 1);
+ max_xfer_regions -=
+ transfer_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId::Collector, max_xfer_regions,
+ collector_xfer);
+ }
+
+ // Process empty regions within the OldCollector free partition
+ if ((max_xfer_regions > 0) &&
+ (_partitions.leftmost_empty(ShenandoahFreeSetPartitionId::OldCollector)
+ <= _partitions.rightmost_empty(ShenandoahFreeSetPartitionId::OldCollector))) {
+ ShenandoahHeapLocker locker(_heap->lock());
+ size_t old_collector_regions =
+ transfer_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId::OldCollector, max_xfer_regions,
+ old_collector_xfer);
+ max_xfer_regions -= old_collector_regions;
+ if (old_collector_regions > 0) {
+ ShenandoahGenerationalHeap::cast(_heap)->generation_sizer()->transfer_to_young(old_collector_regions);
}
}
@@ -1073,81 +1564,202 @@ void ShenandoahFreeSet::move_regions_from_collector_to_mutator(size_t max_xfer_r
if ((max_xfer_regions > 0) && (_partitions.leftmost(ShenandoahFreeSetPartitionId::Collector)
<= _partitions.rightmost(ShenandoahFreeSetPartitionId::Collector))) {
ShenandoahHeapLocker locker(_heap->lock());
- idx_t rightmost = _partitions.rightmost(ShenandoahFreeSetPartitionId::Collector);
- for (idx_t idx = _partitions.leftmost(ShenandoahFreeSetPartitionId::Collector);
- (max_xfer_regions > 0) && (idx <= rightmost); ) {
- assert(_partitions.in_free_set(ShenandoahFreeSetPartitionId::Collector, idx),
- "Boundaries or find_first_set_bit failed: " SSIZE_FORMAT, idx);
- size_t ac = alloc_capacity(idx);
- if (ac > 0) {
- _partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Collector,
- ShenandoahFreeSetPartitionId::Mutator, ac);
- max_xfer_regions--;
- collector_not_empty_xfer += ac;
- }
- idx = _partitions.find_index_of_next_available_region(ShenandoahFreeSetPartitionId::Collector, idx + 1);
- }
+ max_xfer_regions -=
+ transfer_non_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId::Collector, max_xfer_regions,
+ collector_xfer);
}
- size_t collector_xfer = collector_empty_xfer + collector_not_empty_xfer;
- log_info(gc, ergo)("At start of update refs, moving " SIZE_FORMAT "%s to Mutator free partition from Collector Reserve",
- byte_size_in_proper_unit(collector_xfer), proper_unit_for_byte_size(collector_xfer));
+ size_t total_xfer = collector_xfer + old_collector_xfer;
+ log_info(gc, ergo)("At start of update refs, moving " SIZE_FORMAT "%s to Mutator free set from Collector Reserve ("
+ SIZE_FORMAT "%s) and from Old Collector Reserve (" SIZE_FORMAT "%s)",
+ byte_size_in_proper_unit(total_xfer), proper_unit_for_byte_size(total_xfer),
+ byte_size_in_proper_unit(collector_xfer), proper_unit_for_byte_size(collector_xfer),
+ byte_size_in_proper_unit(old_collector_xfer), proper_unit_for_byte_size(old_collector_xfer));
}
-void ShenandoahFreeSet::prepare_to_rebuild(size_t &cset_regions) {
+
+// Overwrite arguments to represent the amount of memory in each generation that is about to be recycled
+void ShenandoahFreeSet::prepare_to_rebuild(size_t &young_cset_regions, size_t &old_cset_regions,
+ size_t &first_old_region, size_t &last_old_region, size_t &old_region_count) {
shenandoah_assert_heaplocked();
+ // This resets all state information, removing all regions from all sets.
+ clear();
+ log_debug(gc, free)("Rebuilding FreeSet");
+
+ // This places regions that have alloc_capacity into the old_collector set if they identify as is_old() or the
+ // mutator set otherwise. All trashed (cset) regions are affiliated young and placed in mutator set.
+ find_regions_with_alloc_capacity(young_cset_regions, old_cset_regions, first_old_region, last_old_region, old_region_count);
+}
- log_debug(gc)("Rebuilding FreeSet");
+void ShenandoahFreeSet::establish_generation_sizes(size_t young_region_count, size_t old_region_count) {
+ assert(young_region_count + old_region_count == ShenandoahHeap::heap()->num_regions(), "Sanity");
+ if (ShenandoahHeap::heap()->mode()->is_generational()) {
+ ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
+ ShenandoahOldGeneration* old_gen = heap->old_generation();
+ ShenandoahYoungGeneration* young_gen = heap->young_generation();
+ size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
- // This places regions that have alloc_capacity into the mutator partition.
- find_regions_with_alloc_capacity(cset_regions);
+ size_t original_old_capacity = old_gen->max_capacity();
+ size_t new_old_capacity = old_region_count * region_size_bytes;
+ size_t new_young_capacity = young_region_count * region_size_bytes;
+ old_gen->set_capacity(new_old_capacity);
+ young_gen->set_capacity(new_young_capacity);
+
+ if (new_old_capacity > original_old_capacity) {
+ size_t region_count = (new_old_capacity - original_old_capacity) / region_size_bytes;
+ log_info(gc, ergo)("Transfer " SIZE_FORMAT " region(s) from %s to %s, yielding increased size: " PROPERFMT,
+ region_count, young_gen->name(), old_gen->name(), PROPERFMTARGS(new_old_capacity));
+ } else if (new_old_capacity < original_old_capacity) {
+ size_t region_count = (original_old_capacity - new_old_capacity) / region_size_bytes;
+ log_info(gc, ergo)("Transfer " SIZE_FORMAT " region(s) from %s to %s, yielding increased size: " PROPERFMT,
+ region_count, old_gen->name(), young_gen->name(), PROPERFMTARGS(new_young_capacity));
+ }
+ // This balances generations, so clear any pending request to balance.
+ old_gen->set_region_balance(0);
+ }
}
-void ShenandoahFreeSet::finish_rebuild(size_t cset_regions) {
+void ShenandoahFreeSet::finish_rebuild(size_t young_cset_regions, size_t old_cset_regions, size_t old_region_count,
+ bool have_evacuation_reserves) {
shenandoah_assert_heaplocked();
+ size_t young_reserve(0), old_reserve(0);
- // Our desire is to reserve this much memory for future evacuation. We may end up reserving less, if
- // memory is in short supply.
-
- size_t reserve = _heap->max_capacity() * ShenandoahEvacReserve / 100;
- size_t available_in_collector_partition = (_partitions.capacity_of(ShenandoahFreeSetPartitionId::Collector)
- - _partitions.used_by(ShenandoahFreeSetPartitionId::Collector));
- size_t additional_reserve;
- if (available_in_collector_partition < reserve) {
- additional_reserve = reserve - available_in_collector_partition;
+ if (_heap->mode()->is_generational()) {
+ compute_young_and_old_reserves(young_cset_regions, old_cset_regions, have_evacuation_reserves,
+ young_reserve, old_reserve);
} else {
- additional_reserve = 0;
+ young_reserve = (_heap->max_capacity() / 100) * ShenandoahEvacReserve;
+ old_reserve = 0;
}
- reserve_regions(reserve);
+ // Move some of the mutator regions in the Collector and OldCollector partitions in order to satisfy
+ // young_reserve and old_reserve.
+ reserve_regions(young_reserve, old_reserve, old_region_count);
+ size_t young_region_count = _heap->num_regions() - old_region_count;
+ establish_generation_sizes(young_region_count, old_region_count);
+ establish_old_collector_alloc_bias();
_partitions.assert_bounds();
log_status();
}
-void ShenandoahFreeSet::rebuild() {
- size_t cset_regions;
- prepare_to_rebuild(cset_regions);
- finish_rebuild(cset_regions);
+void ShenandoahFreeSet::compute_young_and_old_reserves(size_t young_cset_regions, size_t old_cset_regions,
+ bool have_evacuation_reserves,
+ size_t& young_reserve_result, size_t& old_reserve_result) const {
+ shenandoah_assert_generational();
+ const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
+
+ ShenandoahOldGeneration* const old_generation = _heap->old_generation();
+ size_t old_available = old_generation->available();
+ size_t old_unaffiliated_regions = old_generation->free_unaffiliated_regions();
+ ShenandoahYoungGeneration* const young_generation = _heap->young_generation();
+ size_t young_capacity = young_generation->max_capacity();
+ size_t young_unaffiliated_regions = young_generation->free_unaffiliated_regions();
+
+ // Add in the regions we anticipate to be freed by evacuation of the collection set
+ old_unaffiliated_regions += old_cset_regions;
+ young_unaffiliated_regions += young_cset_regions;
+
+ // Consult old-region balance to make adjustments to current generation capacities and availability.
+ // The generation region transfers take place after we rebuild.
+ const ssize_t old_region_balance = old_generation->get_region_balance();
+ if (old_region_balance != 0) {
+#ifdef ASSERT
+ if (old_region_balance > 0) {
+ assert(old_region_balance <= checked_cast(old_unaffiliated_regions), "Cannot transfer regions that are affiliated");
+ } else {
+ assert(0 - old_region_balance <= checked_cast(young_unaffiliated_regions), "Cannot transfer regions that are affiliated");
+ }
+#endif
+
+ ssize_t xfer_bytes = old_region_balance * checked_cast(region_size_bytes);
+ old_available -= xfer_bytes;
+ old_unaffiliated_regions -= old_region_balance;
+ young_capacity += xfer_bytes;
+ young_unaffiliated_regions += old_region_balance;
+ }
+
+ // All allocations taken from the old collector set are performed by GC, generally using PLABs for both
+ // promotions and evacuations. The partition between which old memory is reserved for evacuation and
+ // which is reserved for promotion is enforced using thread-local variables that prescribe intentions for
+ // each PLAB's available memory.
+ if (have_evacuation_reserves) {
+ // We are rebuilding at the end of final mark, having already established evacuation budgets for this GC pass.
+ const size_t promoted_reserve = old_generation->get_promoted_reserve();
+ const size_t old_evac_reserve = old_generation->get_evacuation_reserve();
+ young_reserve_result = young_generation->get_evacuation_reserve();
+ old_reserve_result = promoted_reserve + old_evac_reserve;
+ assert(old_reserve_result <= old_available,
+ "Cannot reserve (" SIZE_FORMAT " + " SIZE_FORMAT") more OLD than is available: " SIZE_FORMAT,
+ promoted_reserve, old_evac_reserve, old_available);
+ } else {
+ // We are rebuilding at end of GC, so we set aside budgets specified on command line (or defaults)
+ young_reserve_result = (young_capacity * ShenandoahEvacReserve) / 100;
+ // The auto-sizer has already made old-gen large enough to hold all anticipated evacuations and promotions.
+ // Affiliated old-gen regions are already in the OldCollector free set. Add in the relevant number of
+ // unaffiliated regions.
+ old_reserve_result = old_available;
+ }
+
+ // Old available regions that have less than PLAB::min_size() of available memory are not placed into the OldCollector
+ // free set. Because of this, old_available may not have enough memory to represent the intended reserve. Adjust
+ // the reserve downward to account for this possibility. This loss is part of the reason why the original budget
+ // was adjusted with ShenandoahOldEvacWaste and ShenandoahOldPromoWaste multipliers.
+ if (old_reserve_result >
+ _partitions.capacity_of(ShenandoahFreeSetPartitionId::OldCollector) + old_unaffiliated_regions * region_size_bytes) {
+ old_reserve_result =
+ _partitions.capacity_of(ShenandoahFreeSetPartitionId::OldCollector) + old_unaffiliated_regions * region_size_bytes;
+ }
+
+ if (young_reserve_result > young_unaffiliated_regions * region_size_bytes) {
+ young_reserve_result = young_unaffiliated_regions * region_size_bytes;
+ }
}
-void ShenandoahFreeSet::reserve_regions(size_t to_reserve) {
+// Having placed all regions that have allocation capacity into the mutator set if they identify as is_young()
+// or into the old collector set if they identify as is_old(), move some of these regions from the mutator set
+// into the collector set or old collector set in order to assure that the memory available for allocations within
+// the collector set is at least to_reserve and the memory available for allocations within the old collector set
+// is at least to_reserve_old.
+void ShenandoahFreeSet::reserve_regions(size_t to_reserve, size_t to_reserve_old, size_t &old_region_count) {
for (size_t i = _heap->num_regions(); i > 0; i--) {
size_t idx = i - 1;
ShenandoahHeapRegion* r = _heap->get_region(idx);
-
if (!_partitions.in_free_set(ShenandoahFreeSetPartitionId::Mutator, idx)) {
continue;
}
size_t ac = alloc_capacity(r);
- assert (ac > 0, "Membership in free partition implies has capacity");
+ assert (ac > 0, "Membership in free set implies has capacity");
+ assert (!r->is_old() || r->is_trash(), "Except for trash, mutator_is_free regions should not be affiliated OLD");
+ bool move_to_old_collector = _partitions.available_in(ShenandoahFreeSetPartitionId::OldCollector) < to_reserve_old;
bool move_to_collector = _partitions.available_in(ShenandoahFreeSetPartitionId::Collector) < to_reserve;
- if (!move_to_collector) {
- // We've satisfied to_reserve
+
+ if (!move_to_collector && !move_to_old_collector) {
+ // We've satisfied both to_reserve and to_reserved_old
break;
}
+ if (move_to_old_collector) {
+ // We give priority to OldCollector partition because we desire to pack OldCollector regions into higher
+ // addresses than Collector regions. Presumably, OldCollector regions are more "stable" and less likely to
+ // be collected in the near future.
+ if (r->is_trash() || !r->is_affiliated()) {
+ // OLD regions that have available memory are already in the old_collector free set.
+ _partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Mutator,
+ ShenandoahFreeSetPartitionId::OldCollector, ac);
+ log_debug(gc)(" Shifting region " SIZE_FORMAT " from mutator_free to old_collector_free", idx);
+ log_debug(gc)(" Shifted Mutator range [" SSIZE_FORMAT ", " SSIZE_FORMAT "],"
+ " Old Collector range [" SSIZE_FORMAT ", " SSIZE_FORMAT "]",
+ _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator),
+ _partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator),
+ _partitions.leftmost(ShenandoahFreeSetPartitionId::OldCollector),
+ _partitions.rightmost(ShenandoahFreeSetPartitionId::OldCollector));
+ old_region_count++;
+ continue;
+ }
+ }
+
if (move_to_collector) {
// Note: In a previous implementation, regions were only placed into the survivor space (collector_is_free) if
// they were entirely empty. This has the effect of causing new Mutator allocation to reside next to objects
@@ -1160,10 +1772,21 @@ void ShenandoahFreeSet::reserve_regions(size_t to_reserve) {
_partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Mutator,
ShenandoahFreeSetPartitionId::Collector, ac);
log_debug(gc)(" Shifting region " SIZE_FORMAT " from mutator_free to collector_free", idx);
+ log_debug(gc)(" Shifted Mutator range [" SSIZE_FORMAT ", " SSIZE_FORMAT "],"
+ " Collector range [" SSIZE_FORMAT ", " SSIZE_FORMAT "]",
+ _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator),
+ _partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator),
+ _partitions.leftmost(ShenandoahFreeSetPartitionId::Collector),
+ _partitions.rightmost(ShenandoahFreeSetPartitionId::Collector));
}
}
if (LogTarget(Info, gc, free)::is_enabled()) {
+ size_t old_reserve = _partitions.available_in(ShenandoahFreeSetPartitionId::OldCollector);
+ if (old_reserve < to_reserve_old) {
+ log_info(gc, free)("Wanted " PROPERFMT " for old reserve, but only reserved: " PROPERFMT,
+ PROPERFMTARGS(to_reserve_old), PROPERFMTARGS(old_reserve));
+ }
size_t reserve = _partitions.available_in(ShenandoahFreeSetPartitionId::Collector);
if (reserve < to_reserve) {
log_debug(gc)("Wanted " PROPERFMT " for young reserve, but only reserved: " PROPERFMT,
@@ -1172,6 +1795,37 @@ void ShenandoahFreeSet::reserve_regions(size_t to_reserve) {
}
}
+void ShenandoahFreeSet::establish_old_collector_alloc_bias() {
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+ shenandoah_assert_heaplocked();
+
+ idx_t left_idx = _partitions.leftmost(ShenandoahFreeSetPartitionId::OldCollector);
+ idx_t right_idx = _partitions.rightmost(ShenandoahFreeSetPartitionId::OldCollector);
+ idx_t middle = (left_idx + right_idx) / 2;
+ size_t available_in_first_half = 0;
+ size_t available_in_second_half = 0;
+
+ for (idx_t index = left_idx; index < middle; index++) {
+ if (_partitions.in_free_set(ShenandoahFreeSetPartitionId::OldCollector, index)) {
+ ShenandoahHeapRegion* r = heap->get_region((size_t) index);
+ available_in_first_half += r->free();
+ }
+ }
+ for (idx_t index = middle; index <= right_idx; index++) {
+ if (_partitions.in_free_set(ShenandoahFreeSetPartitionId::OldCollector, index)) {
+ ShenandoahHeapRegion* r = heap->get_region(index);
+ available_in_second_half += r->free();
+ }
+ }
+
+ // We desire to first consume the sparsely distributed regions in order that the remaining regions are densely packed.
+ // Densely packing regions reduces the effort to search for a region that has sufficient memory to satisfy a new allocation
+ // request. Regions become sparsely distributed following a Full GC, which tends to slide all regions to the front of the
+ // heap rather than allowing survivor regions to remain at the high end of the heap where we intend for them to congregate.
+ _partitions.set_bias_from_left_to_right(ShenandoahFreeSetPartitionId::OldCollector,
+ (available_in_second_half > available_in_first_half));
+}
+
void ShenandoahFreeSet::log_status_under_lock() {
// Must not be heap locked, it acquires heap lock only when log is enabled
shenandoah_assert_not_heaplocked();
@@ -1189,23 +1843,26 @@ void ShenandoahFreeSet::log_status() {
// Dump of the FreeSet details is only enabled if assertions are enabled
if (LogTarget(Debug, gc, free)::is_enabled()) {
#define BUFFER_SIZE 80
- size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
- size_t consumed_collector = 0;
- size_t available_collector = 0;
- size_t consumed_mutator = 0;
- size_t available_mutator = 0;
char buffer[BUFFER_SIZE];
for (uint i = 0; i < BUFFER_SIZE; i++) {
buffer[i] = '\0';
}
- log_debug(gc)("FreeSet map legend: M:mutator_free C:collector_free H:humongous _:retired");
- log_debug(gc)(" mutator free range [" SIZE_FORMAT ".." SIZE_FORMAT "],"
- " collector free range [" SIZE_FORMAT ".." SIZE_FORMAT "]",
+
+ log_debug(gc)("FreeSet map legend:"
+ " M:mutator_free C:collector_free O:old_collector_free"
+ " H:humongous ~:retired old _:retired young");
+ log_debug(gc)(" mutator free range [" SIZE_FORMAT ".." SIZE_FORMAT "] allocating from %s, "
+ " collector free range [" SIZE_FORMAT ".." SIZE_FORMAT "], "
+ "old collector free range [" SIZE_FORMAT ".." SIZE_FORMAT "] allocates from %s",
_partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator),
_partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator),
+ _partitions.alloc_from_left_bias(ShenandoahFreeSetPartitionId::Mutator)? "left to right": "right to left",
_partitions.leftmost(ShenandoahFreeSetPartitionId::Collector),
- _partitions.rightmost(ShenandoahFreeSetPartitionId::Collector));
+ _partitions.rightmost(ShenandoahFreeSetPartitionId::Collector),
+ _partitions.leftmost(ShenandoahFreeSetPartitionId::OldCollector),
+ _partitions.rightmost(ShenandoahFreeSetPartitionId::OldCollector),
+ _partitions.alloc_from_left_bias(ShenandoahFreeSetPartitionId::OldCollector)? "left to right": "right to left");
for (uint i = 0; i < _heap->num_regions(); i++) {
ShenandoahHeapRegion *r = _heap->get_region(i);
@@ -1215,18 +1872,27 @@ void ShenandoahFreeSet::log_status() {
}
if (_partitions.in_free_set(ShenandoahFreeSetPartitionId::Mutator, i)) {
size_t capacity = alloc_capacity(r);
- available_mutator += capacity;
- consumed_mutator += region_size_bytes - capacity;
- buffer[idx] = (capacity == region_size_bytes)? 'M': 'm';
+ assert(!r->is_old() || r->is_trash(), "Old regions except trash regions should not be in mutator_free set");
+ buffer[idx] = (capacity == ShenandoahHeapRegion::region_size_bytes()) ? 'M' : 'm';
} else if (_partitions.in_free_set(ShenandoahFreeSetPartitionId::Collector, i)) {
size_t capacity = alloc_capacity(r);
- available_collector += capacity;
- consumed_collector += region_size_bytes - capacity;
- buffer[idx] = (capacity == region_size_bytes)? 'C': 'c';
+ assert(!r->is_old() || r->is_trash(), "Old regions except trash regions should not be in collector_free set");
+ buffer[idx] = (capacity == ShenandoahHeapRegion::region_size_bytes()) ? 'C' : 'c';
+ } else if (_partitions.in_free_set(ShenandoahFreeSetPartitionId::OldCollector, i)) {
+ size_t capacity = alloc_capacity(r);
+ buffer[idx] = (capacity == ShenandoahHeapRegion::region_size_bytes()) ? 'O' : 'o';
} else if (r->is_humongous()) {
- buffer[idx] = 'h';
+ if (r->is_old()) {
+ buffer[idx] = 'H';
+ } else {
+ buffer[idx] = 'h';
+ }
} else {
- buffer[idx] = '_';
+ if (r->is_old()) {
+ buffer[idx] = '~';
+ } else {
+ buffer[idx] = '_';
+ }
}
}
uint remnant = _heap->num_regions() % 64;
@@ -1284,9 +1950,8 @@ void ShenandoahFreeSet::log_status() {
// retired, the sum of used and capacities within regions that are still in the Mutator free partition may not match
// my internally tracked values of used() and free().
assert(free == total_free, "Free memory should match");
-
ls.print("Free: " SIZE_FORMAT "%s, Max: " SIZE_FORMAT "%s regular, " SIZE_FORMAT "%s humongous, ",
- byte_size_in_proper_unit(free), proper_unit_for_byte_size(free),
+ byte_size_in_proper_unit(total_free), proper_unit_for_byte_size(total_free),
byte_size_in_proper_unit(max), proper_unit_for_byte_size(max),
byte_size_in_proper_unit(max_humongous), proper_unit_for_byte_size(max_humongous)
);
@@ -1333,6 +1998,27 @@ void ShenandoahFreeSet::log_status() {
byte_size_in_proper_unit(max), proper_unit_for_byte_size(max),
byte_size_in_proper_unit(total_used), proper_unit_for_byte_size(total_used));
}
+
+ if (_heap->mode()->is_generational()) {
+ size_t max = 0;
+ size_t total_free = 0;
+ size_t total_used = 0;
+
+ for (idx_t idx = _partitions.leftmost(ShenandoahFreeSetPartitionId::OldCollector);
+ idx <= _partitions.rightmost(ShenandoahFreeSetPartitionId::OldCollector); idx++) {
+ if (_partitions.in_free_set(ShenandoahFreeSetPartitionId::OldCollector, idx)) {
+ ShenandoahHeapRegion *r = _heap->get_region(idx);
+ size_t free = alloc_capacity(r);
+ max = MAX2(max, free);
+ total_free += free;
+ total_used += r->used();
+ }
+ }
+ ls.print_cr(" Old Collector Reserve: " SIZE_FORMAT "%s, Max: " SIZE_FORMAT "%s; Used: " SIZE_FORMAT "%s",
+ byte_size_in_proper_unit(total_free), proper_unit_for_byte_size(total_free),
+ byte_size_in_proper_unit(max), proper_unit_for_byte_size(max),
+ byte_size_in_proper_unit(total_used), proper_unit_for_byte_size(total_used));
+ }
}
}
@@ -1344,6 +2030,7 @@ HeapWord* ShenandoahFreeSet::allocate(ShenandoahAllocRequest& req, bool& in_new_
case ShenandoahAllocRequest::_alloc_shared_gc:
in_new_region = true;
return allocate_contiguous(req);
+ case ShenandoahAllocRequest::_alloc_plab:
case ShenandoahAllocRequest::_alloc_gclab:
case ShenandoahAllocRequest::_alloc_tlab:
in_new_region = false;
@@ -1360,20 +2047,25 @@ HeapWord* ShenandoahFreeSet::allocate(ShenandoahAllocRequest& req, bool& in_new_
void ShenandoahFreeSet::print_on(outputStream* out) const {
out->print_cr("Mutator Free Set: " SIZE_FORMAT "", _partitions.count(ShenandoahFreeSetPartitionId::Mutator));
- idx_t rightmost = _partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator);
- for (idx_t index = _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator); index <= rightmost; ) {
- assert(_partitions.in_free_set(ShenandoahFreeSetPartitionId::Mutator, index),
- "Boundaries or find_first_set_bit failed: " SSIZE_FORMAT, index);
+ ShenandoahLeftRightIterator mutator(const_cast(&_partitions), ShenandoahFreeSetPartitionId::Mutator);
+ for (idx_t index = mutator.current(); mutator.has_next(); index = mutator.next()) {
_heap->get_region(index)->print_on(out);
- index = _partitions.find_index_of_next_available_region(ShenandoahFreeSetPartitionId::Mutator, index + 1);
}
+
out->print_cr("Collector Free Set: " SIZE_FORMAT "", _partitions.count(ShenandoahFreeSetPartitionId::Collector));
- rightmost = _partitions.rightmost(ShenandoahFreeSetPartitionId::Collector);
- for (idx_t index = _partitions.leftmost(ShenandoahFreeSetPartitionId::Collector); index <= rightmost; ) {
- assert(_partitions.in_free_set(ShenandoahFreeSetPartitionId::Collector, index),
- "Boundaries or find_first_set_bit failed: " SSIZE_FORMAT, index);
+ ShenandoahLeftRightIterator collector(const_cast(&_partitions), ShenandoahFreeSetPartitionId::Collector);
+ for (idx_t index = collector.current(); collector.has_next(); index = collector.next()) {
_heap->get_region(index)->print_on(out);
- index = _partitions.find_index_of_next_available_region(ShenandoahFreeSetPartitionId::Collector, index + 1);
+ }
+
+ if (_heap->mode()->is_generational()) {
+ out->print_cr("Old Collector Free Set: " SIZE_FORMAT "", _partitions.count(ShenandoahFreeSetPartitionId::OldCollector));
+ for (idx_t index = _partitions.leftmost(ShenandoahFreeSetPartitionId::OldCollector);
+ index <= _partitions.rightmost(ShenandoahFreeSetPartitionId::OldCollector); index++) {
+ if (_partitions.in_free_set(ShenandoahFreeSetPartitionId::OldCollector, index)) {
+ _heap->get_region(index)->print_on(out);
+ }
+ }
}
}
@@ -1381,15 +2073,12 @@ double ShenandoahFreeSet::internal_fragmentation() {
double squared = 0;
double linear = 0;
- idx_t rightmost = _partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator);
- for (idx_t index = _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator); index <= rightmost; ) {
- assert(_partitions.in_free_set(ShenandoahFreeSetPartitionId::Mutator, index),
- "Boundaries or find_first_set_bit failed: " SSIZE_FORMAT, index);
+ ShenandoahLeftRightIterator iterator(&_partitions, ShenandoahFreeSetPartitionId::Mutator);
+ for (idx_t index = iterator.current(); iterator.has_next(); index = iterator.next()) {
ShenandoahHeapRegion* r = _heap->get_region(index);
size_t used = r->used();
squared += used * used;
linear += used;
- index = _partitions.find_index_of_next_available_region(ShenandoahFreeSetPartitionId::Mutator, index + 1);
}
if (linear > 0) {
@@ -1404,13 +2093,10 @@ double ShenandoahFreeSet::external_fragmentation() {
idx_t last_idx = 0;
size_t max_contig = 0;
size_t empty_contig = 0;
-
size_t free = 0;
- idx_t rightmost = _partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator);
- for (idx_t index = _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator); index <= rightmost; ) {
- assert(_partitions.in_free_set(ShenandoahFreeSetPartitionId::Mutator, index),
- "Boundaries or find_first_set_bit failed: " SSIZE_FORMAT, index);
+ ShenandoahLeftRightIterator iterator(&_partitions, ShenandoahFreeSetPartitionId::Mutator);
+ for (idx_t index = iterator.current(); iterator.has_next(); index = iterator.next()) {
ShenandoahHeapRegion* r = _heap->get_region(index);
if (r->is_empty()) {
free += ShenandoahHeapRegion::region_size_bytes();
@@ -1424,7 +2110,6 @@ double ShenandoahFreeSet::external_fragmentation() {
}
max_contig = MAX2(max_contig, empty_contig);
last_idx = index;
- index = _partitions.find_index_of_next_available_region(ShenandoahFreeSetPartitionId::Mutator, index + 1);
}
if (free > 0) {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp
index e4e2bb4d6e6..c69d73060c9 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp
@@ -34,6 +34,8 @@
enum class ShenandoahFreeSetPartitionId : uint8_t {
Mutator, // Region is in the Mutator free set: available memory is available to mutators.
Collector, // Region is in the Collector free set: available memory is reserved for evacuations.
+ OldCollector, // Region is in the Old Collector free set:
+ // available memory is reserved for old evacuations and for promotions..
NotFree // Region is in no free set: it has no available memory
};
@@ -80,6 +82,10 @@ class ShenandoahRegionPartitions {
size_t _used[UIntNumPartitions];
size_t _region_counts[UIntNumPartitions];
+ // For each partition p, _left_to_right_bias is true iff allocations are normally made from lower indexed regions
+ // before higher indexed regions.
+ bool _left_to_right_bias[UIntNumPartitions];
+
// Shrink the intervals associated with partition when region idx is removed from this free set
inline void shrink_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, ssize_t idx);
@@ -88,6 +94,11 @@ class ShenandoahRegionPartitions {
ssize_t low_idx, ssize_t high_idx);
inline void expand_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, ssize_t idx, size_t capacity);
+ inline bool is_mutator_partition(ShenandoahFreeSetPartitionId p);
+ inline bool is_young_collector_partition(ShenandoahFreeSetPartitionId p);
+ inline bool is_old_collector_partition(ShenandoahFreeSetPartitionId p);
+ inline bool available_implies_empty(size_t available);
+
#ifndef PRODUCT
void dump_bitmap_row(ssize_t region_idx) const;
void dump_bitmap_range(ssize_t start_region_idx, ssize_t end_region_idx) const;
@@ -112,6 +123,13 @@ class ShenandoahRegionPartitions {
ssize_t mutator_leftmost_empty, ssize_t mutator_rightmost_empty,
size_t mutator_region_count, size_t mutator_used);
+ // Set the OldCollector intervals, usage, and capacity according to arguments. We use this at the end of rebuild_free_set()
+ // to avoid the overhead of making many redundant incremental adjustments to the mutator intervals as the free set is being
+ // rebuilt.
+ void establish_old_collector_intervals(ssize_t old_collector_leftmost, ssize_t old_collector_rightmost,
+ ssize_t old_collector_leftmost_empty, ssize_t old_collector_rightmost_empty,
+ size_t old_collector_region_count, size_t old_collector_used);
+
// Retire region idx from within partition, , leaving its capacity and used as part of the original free partition's totals.
// Requires that region idx is in in the Mutator or Collector partitions. Hereafter, identifies this region as NotFree.
// Any remnant of available memory at the time of retirement is added to the original partition's total of used bytes.
@@ -180,6 +198,16 @@ class ShenandoahRegionPartitions {
inline void increase_used(ShenandoahFreeSetPartitionId which_partition, size_t bytes);
+ inline void set_bias_from_left_to_right(ShenandoahFreeSetPartitionId which_partition, bool value) {
+ assert (which_partition < NumPartitions, "selected free set must be valid");
+ _left_to_right_bias[int(which_partition)] = value;
+ }
+
+ inline bool alloc_from_left_bias(ShenandoahFreeSetPartitionId which_partition) const {
+ assert (which_partition < NumPartitions, "selected free set must be valid");
+ return _left_to_right_bias[int(which_partition)];
+ }
+
inline size_t capacity_of(ShenandoahFreeSetPartitionId which_partition) const {
assert (which_partition < NumPartitions, "selected free set must be valid");
return _capacity[int(which_partition)];
@@ -237,7 +265,7 @@ class ShenandoahRegionPartitions {
// The Shenandoah garbage collector evacuates live objects out of specific regions that are identified as members of the
// collection set (cset).
//
-// The ShenandoahFreeSet endeavors to congregrate survivor objects (objects that have been evacuated at least once) at the
+// The ShenandoahFreeSet tries to colocate survivor objects (objects that have been evacuated at least once) at the
// high end of memory. New mutator allocations are taken from the low end of memory. Within the mutator's range of regions,
// humongous allocations are taken from the lowest addresses, and LAB (local allocation buffers) and regular shared allocations
// are taken from the higher address of the mutator's range of regions. This approach allows longer lasting survivor regions
@@ -260,18 +288,22 @@ class ShenandoahFreeSet : public CHeapObj {
ShenandoahRegionPartitions _partitions;
ShenandoahHeapRegion** _trash_regions;
- // Mutator allocations are biased from left-to-right or from right-to-left based on which end of mutator range
- // is most likely to hold partially used regions. In general, we want to finish consuming partially used
- // regions and retire them in order to reduce the regions that must be searched for each allocation request.
- bool _right_to_left_bias;
+ HeapWord* allocate_aligned_plab(size_t size, ShenandoahAllocRequest& req, ShenandoahHeapRegion* r);
+
+ // Return the address of memory allocated, setting in_new_region to true iff the allocation is taken
+ // from a region that was previously empty. Return nullptr if memory could not be allocated.
+ inline HeapWord* allocate_from_partition_with_affiliation(ShenandoahAffiliation affiliation,
+ ShenandoahAllocRequest& req, bool& in_new_region);
// We re-evaluate the left-to-right allocation bias whenever _alloc_bias_weight is less than zero. Each time
// we allocate an object, we decrement the count of this value. Each time we re-evaluate whether to allocate
// from right-to-left or left-to-right, we reset the value of this counter to _InitialAllocBiasWeight.
ssize_t _alloc_bias_weight;
- const ssize_t _InitialAllocBiasWeight = 256;
+ const ssize_t INITIAL_ALLOC_BIAS_WEIGHT = 256;
+ // Increases used memory for the partition if the allocation is successful. `in_new_region` will be set
+ // if this is the first allocation in the region.
HeapWord* try_allocate_in(ShenandoahHeapRegion* region, ShenandoahAllocRequest& req, bool& in_new_region);
// While holding the heap lock, allocate memory for a single object or LAB which is to be entirely contained
@@ -287,11 +319,38 @@ class ShenandoahFreeSet : public CHeapObj {
// Precondition: ShenandoahHeapRegion::requires_humongous(req.size())
HeapWord* allocate_contiguous(ShenandoahAllocRequest& req);
- // Change region r from the Mutator partition to the GC's Collector partition. This requires that the region is entirely empty.
+ // Change region r from the Mutator partition to the GC's Collector or OldCollector partition. This requires that the
+ // region is entirely empty.
+ //
// Typical usage: During evacuation, the GC may find it needs more memory than had been reserved at the start of evacuation to
// hold evacuated objects. If this occurs and memory is still available in the Mutator's free set, we will flip a region from
- // the Mutator free set into the Collector free set.
+ // the Mutator free set into the Collector or OldCollector free set.
void flip_to_gc(ShenandoahHeapRegion* r);
+ void flip_to_old_gc(ShenandoahHeapRegion* r);
+
+ // Handle allocation for mutator.
+ HeapWord* allocate_for_mutator(ShenandoahAllocRequest &req, bool &in_new_region);
+
+ // Update allocation bias and decided whether to allocate from the left or right side of the heap.
+ void update_allocation_bias();
+
+ // Search for regions to satisfy allocation request using iterator.
+ template
+ HeapWord* allocate_from_regions(Iter& iterator, ShenandoahAllocRequest &req, bool &in_new_region);
+
+ // Handle allocation for collector (for evacuation).
+ HeapWord* allocate_for_collector(ShenandoahAllocRequest& req, bool& in_new_region);
+
+ // Search for allocation in region with same affiliation as request, using given iterator.
+ template
+ HeapWord* allocate_with_affiliation(Iter& iterator, ShenandoahAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region);
+
+ // Return true if the respective generation for this request has free regions.
+ bool can_allocate_in_new_region(const ShenandoahAllocRequest& req);
+
+ // Attempt to allocate memory for an evacuation from the mutator's partition.
+ HeapWord* try_allocate_from_mutator(ShenandoahAllocRequest& req, bool& in_new_region);
+
void clear_internal();
void try_recycle_trashed(ShenandoahHeapRegion *r);
@@ -303,21 +362,20 @@ class ShenandoahFreeSet : public CHeapObj {
inline bool has_alloc_capacity(ShenandoahHeapRegion *r) const;
- // This function places all regions that have allocation capacity into the mutator_partition, identifying regions
- // that have no allocation capacity as NotFree. Subsequently, we will move some of the mutator regions into the
- // collector partition with the intent of packing collector memory into the highest (rightmost) addresses of the
- // heap, with mutator memory consuming the lowest addresses of the heap.
- void find_regions_with_alloc_capacity(size_t &cset_regions);
+ size_t transfer_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId which_collector,
+ size_t max_xfer_regions,
+ size_t& bytes_transferred);
+ size_t transfer_non_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId which_collector,
+ size_t max_xfer_regions,
+ size_t& bytes_transferred);
- // Having placed all regions that have allocation capacity into the mutator partition, move some of these regions from
- // the mutator partition into the collector partition in order to assure that the memory available for allocations within
- // the collector partition is at least to_reserve.
- void reserve_regions(size_t to_reserve);
- // Overwrite arguments to represent the number of regions to be reclaimed from the cset
- void prepare_to_rebuild(size_t &cset_regions);
+ // Determine whether we prefer to allocate from left to right or from right to left within the OldCollector free-set.
+ void establish_old_collector_alloc_bias();
- void finish_rebuild(size_t cset_regions);
+ // Set max_capacity for young and old generations
+ void establish_generation_sizes(size_t young_region_count, size_t old_region_count);
+ size_t get_usable_free_words(size_t free_bytes) const;
// log status, assuming lock has already been acquired by the caller.
void log_status();
@@ -330,20 +388,52 @@ class ShenandoahFreeSet : public CHeapObj {
inline size_t alloc_capacity(size_t idx) const;
void clear();
- void rebuild();
+
+ // Examine the existing free set representation, capturing the current state into var arguments:
+ //
+ // young_cset_regions is the number of regions currently in the young cset if we are starting to evacuate, or zero
+ // old_cset_regions is the number of regions currently in the old cset if we are starting a mixed evacuation, or zero
+ // first_old_region is the index of the first region that is part of the OldCollector set
+ // last_old_region is the index of the last region that is part of the OldCollector set
+ // old_region_count is the number of regions in the OldCollector set that have memory available to be allocated
+ void prepare_to_rebuild(size_t &young_cset_regions, size_t &old_cset_regions,
+ size_t &first_old_region, size_t &last_old_region, size_t &old_region_count);
+
+ // At the end of final mark, but before we begin evacuating, heuristics calculate how much memory is required to
+ // hold the results of evacuating to young-gen and to old-gen, and have_evacuation_reserves should be true.
+ // These quantities, stored as reserves for their respective generations, are consulted prior to rebuilding
+ // the free set (ShenandoahFreeSet) in preparation for evacuation. When the free set is rebuilt, we make sure
+ // to reserve sufficient memory in the collector and old_collector sets to hold evacuations.
+ //
+ // We also rebuild the free set at the end of GC, as we prepare to idle GC until the next trigger. In this case,
+ // have_evacuation_reserves is false because we don't yet know how much memory will need to be evacuated in the
+ // next GC cycle. When have_evacuation_reserves is false, the free set rebuild operation reserves for the collector
+ // and old_collector sets based on alternative mechanisms, such as ShenandoahEvacReserve, ShenandoahOldEvacReserve, and
+ // ShenandoahOldCompactionReserve. In a future planned enhancement, the reserve for old_collector set when the
+ // evacuation reserves are unknown, is based in part on anticipated promotion as determined by analysis of live data
+ // found during the previous GC pass which is one less than the current tenure age.
+ //
+ // young_cset_regions is the number of regions currently in the young cset if we are starting to evacuate, or zero
+ // old_cset_regions is the number of regions currently in the old cset if we are starting a mixed evacuation, or zero
+ // num_old_regions is the number of old-gen regions that have available memory for further allocations (excluding old cset)
+ // have_evacuation_reserves is true iff the desired values of young-gen and old-gen evacuation reserves and old-gen
+ // promotion reserve have been precomputed (and can be obtained by invoking
+ // ->get_evacuation_reserve() or old_gen->get_promoted_reserve()
+ void finish_rebuild(size_t young_cset_regions, size_t old_cset_regions, size_t num_old_regions,
+ bool have_evacuation_reserves = false);
+
+ // When a region is promoted in place, we add the region's available memory if it is greater than plab_min_size()
+ // into the old collector partition by invoking this method.
+ void add_promoted_in_place_region_to_old_collector(ShenandoahHeapRegion* region);
// Move up to cset_regions number of regions from being available to the collector to being available to the mutator.
//
// Typical usage: At the end of evacuation, when the collector no longer needs the regions that had been reserved
// for evacuation, invoke this to make regions available for mutator allocations.
- //
- // Note that we plan to replenish the Collector reserve at the end of update refs, at which time all
- // of the regions recycled from the collection set will be available. If the very unlikely event that there
- // are fewer regions in the collection set than remain in the collector set, we limit the transfer in order
- // to assure that the replenished Collector reserve can be sufficiently large.
void move_regions_from_collector_to_mutator(size_t cset_regions);
void recycle_trash();
+
// Acquire heap lock and log status, assuming heap lock is not acquired by the caller.
void log_status_under_lock();
@@ -355,7 +445,6 @@ class ShenandoahFreeSet : public CHeapObj {
}
HeapWord* allocate(ShenandoahAllocRequest& req, bool& in_new_region);
- size_t unsafe_peek_free() const;
/*
* Internal fragmentation metric: describes how fragmented the heap regions are.
@@ -396,6 +485,28 @@ class ShenandoahFreeSet : public CHeapObj {
double external_fragmentation();
void print_on(outputStream* out) const;
+
+ // This function places all regions that have allocation capacity into the mutator partition, or if the region
+ // is already affiliated with old, into the old collector partition, identifying regions that have no allocation
+ // capacity as NotFree. Capture the modified state of the freeset into var arguments:
+ //
+ // young_cset_regions is the number of regions currently in the young cset if we are starting to evacuate, or zero
+ // old_cset_regions is the number of regions currently in the old cset if we are starting a mixed evacuation, or zero
+ // first_old_region is the index of the first region that is part of the OldCollector set
+ // last_old_region is the index of the last region that is part of the OldCollector set
+ // old_region_count is the number of regions in the OldCollector set that have memory available to be allocated
+ void find_regions_with_alloc_capacity(size_t &young_cset_regions, size_t &old_cset_regions,
+ size_t &first_old_region, size_t &last_old_region, size_t &old_region_count);
+
+ // Ensure that Collector has at least to_reserve bytes of available memory, and OldCollector has at least old_reserve
+ // bytes of available memory. On input, old_region_count holds the number of regions already present in the
+ // OldCollector partition. Upon return, old_region_count holds the updated number of regions in the OldCollector partition.
+ void reserve_regions(size_t to_reserve, size_t old_reserve, size_t &old_region_count);
+
+ // Reserve space for evacuations, with regions reserved for old evacuations placed to the right
+ // of regions reserved of young evacuations.
+ void compute_young_and_old_reserves(size_t young_cset_regions, size_t old_cset_regions, bool have_evacuation_reserves,
+ size_t &young_reserve_result, size_t &old_reserve_result) const;
};
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHFREESET_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
index 53cb8e5d20f..3e880271529 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -36,11 +37,15 @@
#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc/shenandoah/shenandoahConcurrentGC.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.hpp"
+#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc/shenandoah/shenandoahFreeSet.hpp"
#include "gc/shenandoah/shenandoahFullGC.hpp"
+#include "gc/shenandoah/shenandoahGenerationalFullGC.hpp"
+#include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
#include "gc/shenandoah/shenandoahMark.inline.hpp"
#include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
+#include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
#include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
@@ -57,7 +62,6 @@
#include "memory/universe.hpp"
#include "oops/compressedOops.inline.hpp"
#include "oops/oop.inline.hpp"
-#include "runtime/javaThread.hpp"
#include "runtime/orderAccess.hpp"
#include "runtime/vmThread.hpp"
#include "utilities/copy.hpp"
@@ -109,6 +113,10 @@ void ShenandoahFullGC::op_full(GCCause::Cause cause) {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
+ if (heap->mode()->is_generational()) {
+ ShenandoahGenerationalFullGC::handle_completion(heap);
+ }
+
metrics.snap_after();
if (metrics.is_good_progress()) {
@@ -120,13 +128,17 @@ void ShenandoahFullGC::op_full(GCCause::Cause cause) {
}
// Regardless if progress was made, we record that we completed a "successful" full GC.
- heap->heuristics()->record_success_full();
+ heap->global_generation()->heuristics()->record_success_full();
heap->shenandoah_policy()->record_success_full();
}
void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
+ if (heap->mode()->is_generational()) {
+ ShenandoahGenerationalFullGC::prepare();
+ }
+
if (ShenandoahVerify) {
heap->verifier()->verify_before_fullgc();
}
@@ -169,10 +181,9 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
}
assert(!heap->is_update_refs_in_progress(), "sanity");
- // b. Cancel concurrent mark, if in progress
+ // b. Cancel all concurrent marks, if in progress
if (heap->is_concurrent_mark_in_progress()) {
- ShenandoahConcurrentGC::cancel();
- heap->set_concurrent_mark_in_progress(false);
+ heap->cancel_concurrent_mark();
}
assert(!heap->is_concurrent_mark_in_progress(), "sanity");
@@ -182,17 +193,21 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
}
// d. Reset the bitmaps for new marking
- heap->reset_mark_bitmap();
+ heap->global_generation()->reset_mark_bitmap();
assert(heap->marking_context()->is_bitmap_clear(), "sanity");
- assert(!heap->marking_context()->is_complete(), "sanity");
+ assert(!heap->global_generation()->is_mark_complete(), "sanity");
// e. Abandon reference discovery and clear all discovered references.
- ShenandoahReferenceProcessor* rp = heap->ref_processor();
+ ShenandoahReferenceProcessor* rp = heap->global_generation()->ref_processor();
rp->abandon_partial_discovery();
// f. Sync pinned region status from the CP marks
heap->sync_pinned_region_status();
+ if (heap->mode()->is_generational()) {
+ ShenandoahGenerationalFullGC::restore_top_before_promote(heap);
+ }
+
// The rest of prologue:
_preserved_marks->init(heap->workers()->active_workers());
@@ -200,6 +215,7 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
}
if (UseTLAB) {
+ // Note: PLABs are also retired with GCLABs in generational mode.
heap->gclabs_retire(ResizeTLAB);
heap->tlabs_retire(ResizeTLAB);
}
@@ -273,12 +289,12 @@ class ShenandoahPrepareForMarkClosure: public ShenandoahHeapRegionClosure {
public:
ShenandoahPrepareForMarkClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
- void heap_region_do(ShenandoahHeapRegion *r) {
+ void heap_region_do(ShenandoahHeapRegion *r) override {
_ctx->capture_top_at_mark_start(r);
r->clear_live_data();
}
- bool is_thread_safe() { return true; }
+ bool is_thread_safe() override { return true; }
};
void ShenandoahFullGC::phase1_mark_heap() {
@@ -287,18 +303,23 @@ void ShenandoahFullGC::phase1_mark_heap() {
ShenandoahHeap* heap = ShenandoahHeap::heap();
- ShenandoahPrepareForMarkClosure cl;
+ ShenandoahPrepareForMarkClosure prepare_for_mark;
+ ShenandoahExcludeRegionClosure cl(&prepare_for_mark);
heap->parallel_heap_region_iterate(&cl);
- heap->set_unload_classes(heap->heuristics()->can_unload_classes());
+ heap->set_unload_classes(heap->global_generation()->heuristics()->can_unload_classes());
- ShenandoahReferenceProcessor* rp = heap->ref_processor();
+ ShenandoahReferenceProcessor* rp = heap->global_generation()->ref_processor();
// enable ("weak") refs discovery
rp->set_soft_reference_policy(true); // forcefully purge all soft references
- ShenandoahSTWMark mark(true /*full_gc*/);
+ ShenandoahSTWMark mark(heap->global_generation(), true /*full_gc*/);
mark.mark();
heap->parallel_cleaning(true /* full_gc */);
+
+ if (ShenandoahHeap::heap()->mode()->is_generational()) {
+ ShenandoahGenerationalFullGC::log_live_in_old(heap);
+ }
}
class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
@@ -426,8 +447,14 @@ void ShenandoahPrepareForCompactionTask::work(uint worker_id) {
GrowableArray empty_regions((int)_heap->num_regions());
- ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
- prepare_for_compaction(cl, empty_regions, it, from_region);
+ if (_heap->mode()->is_generational()) {
+ ShenandoahPrepareForGenerationalCompactionObjectClosure cl(_preserved_marks->get(worker_id),
+ empty_regions, from_region, worker_id);
+ prepare_for_compaction(cl, empty_regions, it, from_region);
+ } else {
+ ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
+ prepare_for_compaction(cl, empty_regions, it, from_region);
+ }
}
template
@@ -474,6 +501,7 @@ void ShenandoahFullGC::calculate_target_humongous_objects() {
size_t to_begin = heap->num_regions();
size_t to_end = heap->num_regions();
+ log_debug(gc)("Full GC calculating target humongous objects from end " SIZE_FORMAT, to_end);
for (size_t c = heap->num_regions(); c > 0; c--) {
ShenandoahHeapRegion *r = heap->get_region(c - 1);
if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
@@ -516,6 +544,7 @@ class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
r->recycle();
}
if (r->is_cset()) {
+ // Leave affiliation unchanged
r->make_regular_bypass();
}
if (r->is_empty_uncommitted()) {
@@ -539,21 +568,18 @@ class ShenandoahTrashImmediateGarbageClosure: public ShenandoahHeapRegionClosure
_heap(ShenandoahHeap::heap()),
_ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
- void heap_region_do(ShenandoahHeapRegion* r) {
+ void heap_region_do(ShenandoahHeapRegion* r) override {
if (r->is_humongous_start()) {
oop humongous_obj = cast_to_oop(r->bottom());
if (!_ctx->is_marked(humongous_obj)) {
- assert(!r->has_live(),
- "Region " SIZE_FORMAT " is not marked, should not have live", r->index());
+ assert(!r->has_live(), "Region " SIZE_FORMAT " is not marked, should not have live", r->index());
_heap->trash_humongous_region_at(r);
} else {
- assert(r->has_live(),
- "Region " SIZE_FORMAT " should have live", r->index());
+ assert(r->has_live(), "Region " SIZE_FORMAT " should have live", r->index());
}
} else if (r->is_humongous_continuation()) {
// If we hit continuation, the non-live humongous starts should have been trashed already
- assert(r->humongous_start_region()->has_live(),
- "Region " SIZE_FORMAT " should have live", r->index());
+ assert(r->humongous_start_region()->has_live(), "Region " SIZE_FORMAT " should have live", r->index());
} else if (r->is_regular()) {
if (!r->has_live()) {
r->make_trash_immediate();
@@ -716,8 +742,9 @@ void ShenandoahFullGC::phase2_calculate_target_addresses(ShenandoahHeapRegionSet
{
// Trash the immediately collectible regions before computing addresses
- ShenandoahTrashImmediateGarbageClosure tigcl;
- heap->heap_region_iterate(&tigcl);
+ ShenandoahTrashImmediateGarbageClosure trash_immediate_garbage;
+ ShenandoahExcludeRegionClosure cl(&trash_immediate_garbage);
+ heap->heap_region_iterate(&cl);
// Make sure regions are in good state: committed, active, clean.
// This is needed because we are potentially sliding the data through them.
@@ -805,6 +832,9 @@ class ShenandoahAdjustPointersTask : public WorkerTask {
if (!r->is_humongous_continuation() && r->has_live()) {
_heap->marked_object_iterate(r, &obj_cl);
}
+ if (_heap->mode()->is_generational()) {
+ ShenandoahGenerationalFullGC::maybe_coalesce_and_fill_region(r);
+ }
r = _regions.next();
}
}
@@ -909,10 +939,21 @@ class ShenandoahCompactObjectsTask : public WorkerTask {
class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
private:
ShenandoahHeap* const _heap;
- size_t _live;
+ bool _is_generational;
+ size_t _young_regions, _young_usage, _young_humongous_waste;
+ size_t _old_regions, _old_usage, _old_humongous_waste;
public:
- ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()), _live(0) {
+ ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()),
+ _is_generational(_heap->mode()->is_generational()),
+ _young_regions(0),
+ _young_usage(0),
+ _young_humongous_waste(0),
+ _old_regions(0),
+ _old_usage(0),
+ _old_humongous_waste(0)
+ {
+ _heap->free_set()->clear();
}
void heap_region_do(ShenandoahHeapRegion* r) {
@@ -931,6 +972,10 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
// Make empty regions that have been allocated into regular
if (r->is_empty() && live > 0) {
+ if (!_is_generational) {
+ r->make_affiliated_maybe();
+ }
+ // else, generational mode compaction has already established affiliation.
r->make_regular_bypass();
if (ZapUnusedHeapArea) {
SpaceMangler::mangle_region(MemRegion(r->top(), r->end()));
@@ -946,15 +991,32 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
if (r->is_trash()) {
live = 0;
r->recycle();
+ } else {
+ if (r->is_old()) {
+ ShenandoahGenerationalFullGC::account_for_region(r, _old_regions, _old_usage, _old_humongous_waste);
+ } else if (r->is_young()) {
+ ShenandoahGenerationalFullGC::account_for_region(r, _young_regions, _young_usage, _young_humongous_waste);
+ }
}
-
r->set_live_data(live);
r->reset_alloc_metadata();
- _live += live;
}
- size_t get_live() {
- return _live;
+ void update_generation_usage() {
+ if (_is_generational) {
+ _heap->old_generation()->establish_usage(_old_regions, _old_usage, _old_humongous_waste);
+ _heap->young_generation()->establish_usage(_young_regions, _young_usage, _young_humongous_waste);
+ } else {
+ assert(_old_regions == 0, "Old regions only expected in generational mode");
+ assert(_old_usage == 0, "Old usage only expected in generational mode");
+ assert(_old_humongous_waste == 0, "Old humongous waste only expected in generational mode");
+ }
+
+ // In generational mode, global usage should be the sum of young and old. This is also true
+ // for non-generational modes except that there are no old regions.
+ _heap->global_generation()->establish_usage(_old_regions + _young_regions,
+ _old_usage + _young_usage,
+ _old_humongous_waste + _young_humongous_waste);
}
};
@@ -985,6 +1047,7 @@ void ShenandoahFullGC::compact_humongous_objects() {
assert(old_start != new_start, "must be real move");
assert(r->is_stw_move_allowed(), "Region " SIZE_FORMAT " should be movable", r->index());
+ log_debug(gc)("Full GC compaction moves humongous object from region " SIZE_FORMAT " to region " SIZE_FORMAT, old_start, new_start);
Copy::aligned_conjoint_words(r->bottom(), heap->get_region(new_start)->bottom(), words_size);
ContinuationGCSupport::relativize_stack_chunk(cast_to_oop(r->bottom()));
@@ -992,8 +1055,10 @@ void ShenandoahFullGC::compact_humongous_objects() {
new_obj->init_mark();
{
+ ShenandoahAffiliation original_affiliation = r->affiliation();
for (size_t c = old_start; c <= old_end; c++) {
ShenandoahHeapRegion* r = heap->get_region(c);
+ // Leave humongous region affiliation unchanged.
r->make_regular_bypass();
r->set_top(r->bottom());
}
@@ -1001,9 +1066,9 @@ void ShenandoahFullGC::compact_humongous_objects() {
for (size_t c = new_start; c <= new_end; c++) {
ShenandoahHeapRegion* r = heap->get_region(c);
if (c == new_start) {
- r->make_humongous_start_bypass();
+ r->make_humongous_start_bypass(original_affiliation);
} else {
- r->make_humongous_cont_bypass();
+ r->make_humongous_cont_bypass(original_affiliation);
}
// Trailing region may be non-full, record the remainder there
@@ -1088,13 +1153,35 @@ void ShenandoahFullGC::phase5_epilog() {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild);
ShenandoahPostCompactClosure post_compact;
heap->heap_region_iterate(&post_compact);
- heap->set_used(post_compact.get_live());
+ post_compact.update_generation_usage();
+
+ if (heap->mode()->is_generational()) {
+ ShenandoahGenerationalFullGC::balance_generations_after_gc(heap);
+ }
heap->collection_set()->clear();
- heap->free_set()->rebuild();
- heap->clear_cancelled_gc();
+ size_t young_cset_regions, old_cset_regions;
+ size_t first_old, last_old, num_old;
+ heap->free_set()->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old);
+
+ // We also do not expand old generation size following Full GC because we have scrambled age populations and
+ // no longer have objects separated by age into distinct regions.
+ if (heap->mode()->is_generational()) {
+ ShenandoahGenerationalFullGC::compute_balances();
+ }
+
+ heap->free_set()->finish_rebuild(young_cset_regions, old_cset_regions, num_old);
+
+ heap->clear_cancelled_gc(true /* clear oom handler */);
}
_preserved_marks->restore(heap->workers());
_preserved_marks->reclaim();
+
+ // We defer generation resizing actions until after cset regions have been recycled. We do this even following an
+ // abbreviated cycle.
+ if (heap->mode()->is_generational()) {
+ ShenandoahGenerationalFullGC::balance_generations_after_rebuilding_free_set();
+ ShenandoahGenerationalFullGC::rebuild_remembered_set(heap);
+ }
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp
index 719abde4b16..99ee88d98d1 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp
@@ -39,6 +39,8 @@ const char* ShenandoahGC::degen_point_to_string(ShenandoahDegenPoint point) {
return "";
case _degenerated_outside_cycle:
return "Outside of Cycle";
+ case _degenerated_roots:
+ return "Roots";
case _degenerated_mark:
return "Mark";
case _degenerated_evac:
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGC.hpp
index a1714b52372..37b22848917 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahGC.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGC.hpp
@@ -50,6 +50,7 @@ class ShenandoahGC : public StackObj {
enum ShenandoahDegenPoint {
_degenerated_unset,
_degenerated_outside_cycle,
+ _degenerated_roots,
_degenerated_mark,
_degenerated_evac,
_degenerated_updaterefs,
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp
new file mode 100644
index 00000000000..1c644a9accc
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp
@@ -0,0 +1,1035 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
+#include "gc/shenandoah/shenandoahCollectionSetPreselector.hpp"
+#include "gc/shenandoah/shenandoahFreeSet.hpp"
+#include "gc/shenandoah/shenandoahGeneration.hpp"
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
+#include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
+#include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
+#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
+#include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
+#include "gc/shenandoah/shenandoahUtils.hpp"
+#include "gc/shenandoah/shenandoahVerifier.hpp"
+#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
+#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+
+#include "utilities/quickSort.hpp"
+
+
+class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
+private:
+ ShenandoahHeap* _heap;
+ ShenandoahMarkingContext* const _ctx;
+public:
+ ShenandoahResetUpdateRegionStateClosure() :
+ _heap(ShenandoahHeap::heap()),
+ _ctx(_heap->marking_context()) {}
+
+ void heap_region_do(ShenandoahHeapRegion* r) override {
+ if (r->is_active()) {
+ // Reset live data and set TAMS optimistically. We would recheck these under the pause
+ // anyway to capture any updates that happened since now.
+ _ctx->capture_top_at_mark_start(r);
+ r->clear_live_data();
+ }
+ }
+
+ bool is_thread_safe() override { return true; }
+};
+
+class ShenandoahResetBitmapTask : public WorkerTask {
+private:
+ ShenandoahRegionIterator _regions;
+ ShenandoahGeneration* _generation;
+
+public:
+ ShenandoahResetBitmapTask(ShenandoahGeneration* generation) :
+ WorkerTask("Shenandoah Reset Bitmap"), _generation(generation) {}
+
+ void work(uint worker_id) {
+ ShenandoahHeapRegion* region = _regions.next();
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+ ShenandoahMarkingContext* const ctx = heap->marking_context();
+ while (region != nullptr) {
+ auto const affiliation = region->affiliation();
+ bool needs_reset = affiliation == FREE || _generation->contains(affiliation);
+ if (needs_reset && heap->is_bitmap_slice_committed(region)) {
+ ctx->clear_bitmap(region);
+ }
+ region = _regions.next();
+ }
+ }
+};
+
+// Copy the write-version of the card-table into the read-version, clearing the
+// write-copy.
+class ShenandoahMergeWriteTable: public ShenandoahHeapRegionClosure {
+private:
+ ShenandoahScanRemembered* _scanner;
+public:
+ ShenandoahMergeWriteTable(ShenandoahScanRemembered* scanner) : _scanner(scanner) {}
+
+ void heap_region_do(ShenandoahHeapRegion* r) override {
+ assert(r->is_old(), "Don't waste time doing this for non-old regions");
+ _scanner->merge_write_table(r->bottom(), ShenandoahHeapRegion::region_size_words());
+ }
+
+ bool is_thread_safe() override {
+ return true;
+ }
+};
+
+class ShenandoahCopyWriteCardTableToRead: public ShenandoahHeapRegionClosure {
+private:
+ ShenandoahScanRemembered* _scanner;
+public:
+ ShenandoahCopyWriteCardTableToRead(ShenandoahScanRemembered* scanner) : _scanner(scanner) {}
+
+ void heap_region_do(ShenandoahHeapRegion* region) override {
+ assert(region->is_old(), "Don't waste time doing this for non-old regions");
+ _scanner->reset_remset(region->bottom(), ShenandoahHeapRegion::region_size_words());
+ }
+
+ bool is_thread_safe() override { return true; }
+};
+
+// Add [TAMS, top) volume over young regions. Used to correct age 0 cohort census
+// for adaptive tenuring when census is taken during marking.
+// In non-product builds, for the purposes of verification, we also collect the total
+// live objects in young regions as well.
+class ShenandoahUpdateCensusZeroCohortClosure : public ShenandoahHeapRegionClosure {
+private:
+ ShenandoahMarkingContext* const _ctx;
+ // Population size units are words (not bytes)
+ size_t _age0_pop; // running tally of age0 population size
+ size_t _total_pop; // total live population size
+public:
+ explicit ShenandoahUpdateCensusZeroCohortClosure(ShenandoahMarkingContext* ctx)
+ : _ctx(ctx), _age0_pop(0), _total_pop(0) {}
+
+ void heap_region_do(ShenandoahHeapRegion* r) override {
+ if (_ctx != nullptr && r->is_active()) {
+ assert(r->is_young(), "Young regions only");
+ HeapWord* tams = _ctx->top_at_mark_start(r);
+ HeapWord* top = r->top();
+ if (top > tams) {
+ _age0_pop += pointer_delta(top, tams);
+ }
+ // TODO: check significance of _ctx != nullptr above, can that
+ // spoof _total_pop in some corner cases?
+ NOT_PRODUCT(_total_pop += r->get_live_data_words();)
+ }
+ }
+
+ size_t get_age0_population() const { return _age0_pop; }
+ size_t get_total_population() const { return _total_pop; }
+};
+
+void ShenandoahGeneration::confirm_heuristics_mode() {
+ if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
+ vm_exit_during_initialization(
+ err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
+ _heuristics->name()));
+ }
+ if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
+ vm_exit_during_initialization(
+ err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
+ _heuristics->name()));
+ }
+}
+
+ShenandoahHeuristics* ShenandoahGeneration::initialize_heuristics(ShenandoahMode* gc_mode) {
+ _heuristics = gc_mode->initialize_heuristics(this);
+ _heuristics->set_guaranteed_gc_interval(ShenandoahGuaranteedGCInterval);
+ confirm_heuristics_mode();
+ return _heuristics;
+}
+
+size_t ShenandoahGeneration::bytes_allocated_since_gc_start() const {
+ return Atomic::load(&_bytes_allocated_since_gc_start);
+}
+
+void ShenandoahGeneration::reset_bytes_allocated_since_gc_start() {
+ Atomic::store(&_bytes_allocated_since_gc_start, (size_t)0);
+}
+
+void ShenandoahGeneration::increase_allocated(size_t bytes) {
+ Atomic::add(&_bytes_allocated_since_gc_start, bytes, memory_order_relaxed);
+}
+
+void ShenandoahGeneration::set_evacuation_reserve(size_t new_val) {
+ _evacuation_reserve = new_val;
+}
+
+size_t ShenandoahGeneration::get_evacuation_reserve() const {
+ return _evacuation_reserve;
+}
+
+void ShenandoahGeneration::augment_evacuation_reserve(size_t increment) {
+ _evacuation_reserve += increment;
+}
+
+void ShenandoahGeneration::log_status(const char *msg) const {
+ typedef LogTarget(Info, gc, ergo) LogGcInfo;
+
+ if (!LogGcInfo::is_enabled()) {
+ return;
+ }
+
+ // Not under a lock here, so read each of these once to make sure
+ // byte size in proper unit and proper unit for byte size are consistent.
+ size_t v_used = used();
+ size_t v_used_regions = used_regions_size();
+ size_t v_soft_max_capacity = soft_max_capacity();
+ size_t v_max_capacity = max_capacity();
+ size_t v_available = available();
+ size_t v_humongous_waste = get_humongous_waste();
+ LogGcInfo::print("%s: %s generation used: " SIZE_FORMAT "%s, used regions: " SIZE_FORMAT "%s, "
+ "humongous waste: " SIZE_FORMAT "%s, soft capacity: " SIZE_FORMAT "%s, max capacity: " SIZE_FORMAT "%s, "
+ "available: " SIZE_FORMAT "%s", msg, name(),
+ byte_size_in_proper_unit(v_used), proper_unit_for_byte_size(v_used),
+ byte_size_in_proper_unit(v_used_regions), proper_unit_for_byte_size(v_used_regions),
+ byte_size_in_proper_unit(v_humongous_waste), proper_unit_for_byte_size(v_humongous_waste),
+ byte_size_in_proper_unit(v_soft_max_capacity), proper_unit_for_byte_size(v_soft_max_capacity),
+ byte_size_in_proper_unit(v_max_capacity), proper_unit_for_byte_size(v_max_capacity),
+ byte_size_in_proper_unit(v_available), proper_unit_for_byte_size(v_available));
+}
+
+void ShenandoahGeneration::reset_mark_bitmap() {
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+ heap->assert_gc_workers(heap->workers()->active_workers());
+
+ set_mark_incomplete();
+
+ ShenandoahResetBitmapTask task(this);
+ heap->workers()->run_task(&task);
+}
+
+// The ideal is to swap the remembered set so the safepoint effort is no more than a few pointer manipulations.
+// However, limitations in the implementation of the mutator write-barrier make it difficult to simply change the
+// location of the card table. So the interim implementation of swap_remembered_set will copy the write-table
+// onto the read-table and will then clear the write-table.
+void ShenandoahGeneration::swap_remembered_set() {
+ // Must be sure that marking is complete before we swap remembered set.
+ ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
+ heap->assert_gc_workers(heap->workers()->active_workers());
+ shenandoah_assert_safepoint();
+
+ ShenandoahOldGeneration* old_generation = heap->old_generation();
+ ShenandoahCopyWriteCardTableToRead task(old_generation->card_scan());
+ old_generation->parallel_heap_region_iterate(&task);
+}
+
+// Copy the write-version of the card-table into the read-version, clearing the
+// write-version. The work is done at a safepoint and in parallel by the GC
+// worker threads.
+void ShenandoahGeneration::merge_write_table() {
+ // This should only happen for degenerated cycles
+ ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
+ heap->assert_gc_workers(heap->workers()->active_workers());
+ shenandoah_assert_safepoint();
+
+ ShenandoahOldGeneration* old_generation = heap->old_generation();
+ ShenandoahMergeWriteTable task(old_generation->card_scan());
+ old_generation->parallel_heap_region_iterate(&task);
+}
+
+void ShenandoahGeneration::prepare_gc() {
+
+ reset_mark_bitmap();
+
+ // Capture Top At Mark Start for this generation (typically young) and reset mark bitmap.
+ ShenandoahResetUpdateRegionStateClosure cl;
+ parallel_heap_region_iterate_free(&cl);
+}
+
+void ShenandoahGeneration::parallel_heap_region_iterate_free(ShenandoahHeapRegionClosure* cl) {
+ ShenandoahHeap::heap()->parallel_heap_region_iterate(cl);
+}
+
+void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* const heap) {
+ shenandoah_assert_generational();
+
+ ShenandoahOldGeneration* const old_generation = heap->old_generation();
+ ShenandoahYoungGeneration* const young_generation = heap->young_generation();
+
+ // During initialization and phase changes, it is more likely that fewer objects die young and old-gen
+ // memory is not yet full (or is in the process of being replaced). During these times especially, it
+ // is beneficial to loan memory from old-gen to young-gen during the evacuation and update-refs phases
+ // of execution.
+
+ // Calculate EvacuationReserve before PromotionReserve. Evacuation is more critical than promotion.
+ // If we cannot evacuate old-gen, we will not be able to reclaim old-gen memory. Promotions are less
+ // critical. If we cannot promote, there may be degradation of young-gen memory because old objects
+ // accumulate there until they can be promoted. This increases the young-gen marking and evacuation work.
+
+ // First priority is to reclaim the easy garbage out of young-gen.
+
+ // maximum_young_evacuation_reserve is upper bound on memory to be evacuated out of young
+ const size_t maximum_young_evacuation_reserve = (young_generation->max_capacity() * ShenandoahEvacReserve) / 100;
+ const size_t young_evacuation_reserve = MIN2(maximum_young_evacuation_reserve, young_generation->available_with_reserve());
+
+ // maximum_old_evacuation_reserve is an upper bound on memory evacuated from old and evacuated to old (promoted),
+ // clamped by the old generation space available.
+ //
+ // Here's the algebra.
+ // Let SOEP = ShenandoahOldEvacRatioPercent,
+ // OE = old evac,
+ // YE = young evac, and
+ // TE = total evac = OE + YE
+ // By definition:
+ // SOEP/100 = OE/TE
+ // = OE/(OE+YE)
+ // => SOEP/(100-SOEP) = OE/((OE+YE)-OE) // componendo-dividendo: If a/b = c/d, then a/(b-a) = c/(d-c)
+ // = OE/YE
+ // => OE = YE*SOEP/(100-SOEP)
+
+ // We have to be careful in the event that SOEP is set to 100 by the user.
+ assert(ShenandoahOldEvacRatioPercent <= 100, "Error");
+ const size_t old_available = old_generation->available();
+ const size_t maximum_old_evacuation_reserve = (ShenandoahOldEvacRatioPercent == 100) ?
+ old_available : MIN2((maximum_young_evacuation_reserve * ShenandoahOldEvacRatioPercent) / (100 - ShenandoahOldEvacRatioPercent),
+ old_available);
+
+
+ // Second priority is to reclaim garbage out of old-gen if there are old-gen collection candidates. Third priority
+ // is to promote as much as we have room to promote. However, if old-gen memory is in short supply, this means young
+ // GC is operating under "duress" and was unable to transfer the memory that we would normally expect. In this case,
+ // old-gen will refrain from compacting itself in order to allow a quicker young-gen cycle (by avoiding the update-refs
+ // through ALL of old-gen). If there is some memory available in old-gen, we will use this for promotions as promotions
+ // do not add to the update-refs burden of GC.
+
+ size_t old_evacuation_reserve, old_promo_reserve;
+ if (is_global()) {
+ // Global GC is typically triggered by user invocation of System.gc(), and typically indicates that there is lots
+ // of garbage to be reclaimed because we are starting a new phase of execution. Marking for global GC may take
+ // significantly longer than typical young marking because we must mark through all old objects. To expedite
+ // evacuation and update-refs, we give emphasis to reclaiming garbage first, wherever that garbage is found.
+ // Global GC will adjust generation sizes to accommodate the collection set it chooses.
+
+ // Set old_promo_reserve to enforce that no regions are preselected for promotion. Such regions typically
+ // have relatively high memory utilization. We still call select_aged_regions() because this will prepare for
+ // promotions in place, if relevant.
+ old_promo_reserve = 0;
+
+ // Dedicate all available old memory to old_evacuation reserve. This may be small, because old-gen is only
+ // expanded based on an existing mixed evacuation workload at the end of the previous GC cycle. We'll expand
+ // the budget for evacuation of old during GLOBAL cset selection.
+ old_evacuation_reserve = maximum_old_evacuation_reserve;
+ } else if (old_generation->has_unprocessed_collection_candidates()) {
+ // We reserved all old-gen memory at end of previous GC to hold anticipated evacuations to old-gen. If this is
+ // mixed evacuation, reserve all of this memory for compaction of old-gen and do not promote. Prioritize compaction
+ // over promotion in order to defragment OLD so that it will be better prepared to efficiently receive promoted memory.
+ old_evacuation_reserve = maximum_old_evacuation_reserve;
+ old_promo_reserve = 0;
+ } else {
+ // Make all old-evacuation memory for promotion, but if we can't use it all for promotion, we'll allow some evacuation.
+ old_evacuation_reserve = 0;
+ old_promo_reserve = maximum_old_evacuation_reserve;
+ }
+ assert(old_evacuation_reserve <= old_available, "Error");
+
+ // We see too many old-evacuation failures if we force ourselves to evacuate into regions that are not initially empty.
+ // So we limit the old-evacuation reserve to unfragmented memory. Even so, old-evacuation is free to fill in nooks and
+ // crannies within existing partially used regions and it generally tries to do so.
+ const size_t old_free_unfragmented = old_generation->free_unaffiliated_regions() * ShenandoahHeapRegion::region_size_bytes();
+ if (old_evacuation_reserve > old_free_unfragmented) {
+ const size_t delta = old_evacuation_reserve - old_free_unfragmented;
+ old_evacuation_reserve -= delta;
+ // Let promo consume fragments of old-gen memory if not global
+ if (!is_global()) {
+ old_promo_reserve += delta;
+ }
+ }
+
+ // Preselect regions for promotion by evacuation (obtaining the live data to seed promoted_reserve),
+ // and identify regions that will promote in place. These use the tenuring threshold.
+ const size_t consumed_by_advance_promotion = select_aged_regions(old_promo_reserve);
+ assert(consumed_by_advance_promotion <= maximum_old_evacuation_reserve, "Cannot promote more than available old-gen memory");
+
+ // Note that unused old_promo_reserve might not be entirely consumed_by_advance_promotion. Do not transfer this
+ // to old_evacuation_reserve because this memory is likely very fragmented, and we do not want to increase the likelihood
+ // of old evacuation failure.
+ young_generation->set_evacuation_reserve(young_evacuation_reserve);
+ old_generation->set_evacuation_reserve(old_evacuation_reserve);
+ old_generation->set_promoted_reserve(consumed_by_advance_promotion);
+
+ // There is no need to expand OLD because all memory used here was set aside at end of previous GC, except in the
+ // case of a GLOBAL gc. During choose_collection_set() of GLOBAL, old will be expanded on demand.
+}
+
+// Having chosen the collection set, adjust the budgets for generational mode based on its composition. Note
+// that young_generation->available() now knows about recently discovered immediate garbage.
+//
+void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* const heap, ShenandoahCollectionSet* const collection_set) {
+ shenandoah_assert_generational();
+ // We may find that old_evacuation_reserve and/or loaned_for_young_evacuation are not fully consumed, in which case we may
+ // be able to increase regions_available_to_loan
+
+ // The role of adjust_evacuation_budgets() is to compute the correct value of regions_available_to_loan and to make
+ // effective use of this memory, including the remnant memory within these regions that may result from rounding loan to
+ // integral number of regions. Excess memory that is available to be loaned is applied to an allocation supplement,
+ // which allows mutators to allocate memory beyond the current capacity of young-gen on the promise that the loan
+ // will be repaid as soon as we finish updating references for the recently evacuated collection set.
+
+ // We cannot recalculate regions_available_to_loan by simply dividing old_generation->available() by region_size_bytes
+ // because the available memory may be distributed between many partially occupied regions that are already holding old-gen
+ // objects. Memory in partially occupied regions is not "available" to be loaned. Note that an increase in old-gen
+ // available that results from a decrease in memory consumed by old evacuation is not necessarily available to be loaned
+ // to young-gen.
+
+ size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
+ ShenandoahOldGeneration* const old_generation = heap->old_generation();
+ ShenandoahYoungGeneration* const young_generation = heap->young_generation();
+
+ size_t old_evacuated = collection_set->get_old_bytes_reserved_for_evacuation();
+ size_t old_evacuated_committed = (size_t) (ShenandoahOldEvacWaste * double(old_evacuated));
+ size_t old_evacuation_reserve = old_generation->get_evacuation_reserve();
+
+ if (old_evacuated_committed > old_evacuation_reserve) {
+ // This should only happen due to round-off errors when enforcing ShenandoahOldEvacWaste
+ assert(old_evacuated_committed <= (33 * old_evacuation_reserve) / 32,
+ "Round-off errors should be less than 3.125%%, committed: " SIZE_FORMAT ", reserved: " SIZE_FORMAT,
+ old_evacuated_committed, old_evacuation_reserve);
+ old_evacuated_committed = old_evacuation_reserve;
+ // Leave old_evac_reserve as previously configured
+ } else if (old_evacuated_committed < old_evacuation_reserve) {
+ // This happens if the old-gen collection consumes less than full budget.
+ old_evacuation_reserve = old_evacuated_committed;
+ old_generation->set_evacuation_reserve(old_evacuation_reserve);
+ }
+
+ size_t young_advance_promoted = collection_set->get_young_bytes_to_be_promoted();
+ size_t young_advance_promoted_reserve_used = (size_t) (ShenandoahPromoEvacWaste * double(young_advance_promoted));
+
+ size_t young_evacuated = collection_set->get_young_bytes_reserved_for_evacuation();
+ size_t young_evacuated_reserve_used = (size_t) (ShenandoahEvacWaste * double(young_evacuated));
+
+ size_t total_young_available = young_generation->available_with_reserve();
+ assert(young_evacuated_reserve_used <= total_young_available, "Cannot evacuate more than is available in young");
+ young_generation->set_evacuation_reserve(young_evacuated_reserve_used);
+
+ size_t old_available = old_generation->available();
+ // Now that we've established the collection set, we know how much memory is really required by old-gen for evacuation
+ // and promotion reserves. Try shrinking OLD now in case that gives us a bit more runway for mutator allocations during
+ // evac and update phases.
+ size_t old_consumed = old_evacuated_committed + young_advance_promoted_reserve_used;
+
+ if (old_available < old_consumed) {
+ // This can happen due to round-off errors when adding the results of truncated integer arithmetic.
+ // We've already truncated old_evacuated_committed. Truncate young_advance_promoted_reserve_used here.
+ assert(young_advance_promoted_reserve_used <= (33 * (old_available - old_evacuated_committed)) / 32,
+ "Round-off errors should be less than 3.125%%, committed: " SIZE_FORMAT ", reserved: " SIZE_FORMAT,
+ young_advance_promoted_reserve_used, old_available - old_evacuated_committed);
+ young_advance_promoted_reserve_used = old_available - old_evacuated_committed;
+ old_consumed = old_evacuated_committed + young_advance_promoted_reserve_used;
+ }
+
+ assert(old_available >= old_consumed, "Cannot consume (" SIZE_FORMAT ") more than is available (" SIZE_FORMAT ")",
+ old_consumed, old_available);
+ size_t excess_old = old_available - old_consumed;
+ size_t unaffiliated_old_regions = old_generation->free_unaffiliated_regions();
+ size_t unaffiliated_old = unaffiliated_old_regions * region_size_bytes;
+ assert(old_available >= unaffiliated_old, "Unaffiliated old is a subset of old available");
+
+ // Make sure old_evac_committed is unaffiliated
+ if (old_evacuated_committed > 0) {
+ if (unaffiliated_old > old_evacuated_committed) {
+ size_t giveaway = unaffiliated_old - old_evacuated_committed;
+ size_t giveaway_regions = giveaway / region_size_bytes; // round down
+ if (giveaway_regions > 0) {
+ excess_old = MIN2(excess_old, giveaway_regions * region_size_bytes);
+ } else {
+ excess_old = 0;
+ }
+ } else {
+ excess_old = 0;
+ }
+ }
+
+ // If we find that OLD has excess regions, give them back to YOUNG now to reduce likelihood we run out of allocation
+ // runway during evacuation and update-refs.
+ size_t regions_to_xfer = 0;
+ if (excess_old > unaffiliated_old) {
+ // we can give back unaffiliated_old (all of unaffiliated is excess)
+ if (unaffiliated_old_regions > 0) {
+ regions_to_xfer = unaffiliated_old_regions;
+ }
+ } else if (unaffiliated_old_regions > 0) {
+ // excess_old < unaffiliated old: we can give back MIN(excess_old/region_size_bytes, unaffiliated_old_regions)
+ size_t excess_regions = excess_old / region_size_bytes;
+ regions_to_xfer = MIN2(excess_regions, unaffiliated_old_regions);
+ }
+
+ if (regions_to_xfer > 0) {
+ bool result = ShenandoahGenerationalHeap::cast(heap)->generation_sizer()->transfer_to_young(regions_to_xfer);
+ assert(excess_old >= regions_to_xfer * region_size_bytes,
+ "Cannot transfer (" SIZE_FORMAT ", " SIZE_FORMAT ") more than excess old (" SIZE_FORMAT ")",
+ regions_to_xfer, region_size_bytes, excess_old);
+ excess_old -= regions_to_xfer * region_size_bytes;
+ log_debug(gc, ergo)("%s transferred " SIZE_FORMAT " excess regions to young before start of evacuation",
+ result? "Successfully": "Unsuccessfully", regions_to_xfer);
+ }
+
+ // Add in the excess_old memory to hold unanticipated promotions, if any. If there are more unanticipated
+ // promotions than fit in reserved memory, they will be deferred until a future GC pass.
+ size_t total_promotion_reserve = young_advance_promoted_reserve_used + excess_old;
+ old_generation->set_promoted_reserve(total_promotion_reserve);
+ old_generation->reset_promoted_expended();
+}
+
+typedef struct {
+ ShenandoahHeapRegion* _region;
+ size_t _live_data;
+} AgedRegionData;
+
+static int compare_by_aged_live(AgedRegionData a, AgedRegionData b) {
+ if (a._live_data < b._live_data)
+ return -1;
+ else if (a._live_data > b._live_data)
+ return 1;
+ else return 0;
+}
+
+inline void assert_no_in_place_promotions() {
+#ifdef ASSERT
+ class ShenandoahNoInPlacePromotions : public ShenandoahHeapRegionClosure {
+ public:
+ void heap_region_do(ShenandoahHeapRegion *r) override {
+ assert(r->get_top_before_promote() == nullptr,
+ "Region " SIZE_FORMAT " should not be ready for in-place promotion", r->index());
+ }
+ } cl;
+ ShenandoahHeap::heap()->heap_region_iterate(&cl);
+#endif
+}
+
+// Preselect for inclusion into the collection set regions whose age is at or above tenure age which contain more than
+// ShenandoahOldGarbageThreshold amounts of garbage. We identify these regions by setting the appropriate entry of
+// the collection set's preselected regions array to true. All entries are initialized to false before calling this
+// function.
+//
+// During the subsequent selection of the collection set, we give priority to these promotion set candidates.
+// Without this prioritization, we found that the aged regions tend to be ignored because they typically have
+// much less garbage and much more live data than the recently allocated "eden" regions. When aged regions are
+// repeatedly excluded from the collection set, the amount of live memory within the young generation tends to
+// accumulate and this has the undesirable side effect of causing young-generation collections to require much more
+// CPU and wall-clock time.
+//
+// A second benefit of treating aged regions differently than other regions during collection set selection is
+// that this allows us to more accurately budget memory to hold the results of evacuation. Memory for evacuation
+// of aged regions must be reserved in the old generation. Memory for evacuation of all other regions must be
+// reserved in the young generation.
+size_t ShenandoahGeneration::select_aged_regions(size_t old_available) {
+
+ // There should be no regions configured for subsequent in-place-promotions carried over from the previous cycle.
+ assert_no_in_place_promotions();
+
+ auto const heap = ShenandoahGenerationalHeap::heap();
+ bool* const candidate_regions_for_promotion_by_copy = heap->collection_set()->preselected_regions();
+ ShenandoahMarkingContext* const ctx = heap->marking_context();
+
+ const uint tenuring_threshold = heap->age_census()->tenuring_threshold();
+ const size_t old_garbage_threshold = (ShenandoahHeapRegion::region_size_bytes() * ShenandoahOldGarbageThreshold) / 100;
+
+ size_t old_consumed = 0;
+ size_t promo_potential = 0;
+ size_t candidates = 0;
+
+ // Tracks the padding of space above top in regions eligible for promotion in place
+ size_t promote_in_place_pad = 0;
+
+ // Sort the promotion-eligible regions in order of increasing live-data-bytes so that we can first reclaim regions that require
+ // less evacuation effort. This prioritizes garbage first, expanding the allocation pool early before we reclaim regions that
+ // have more live data.
+ const size_t num_regions = heap->num_regions();
+
+ ResourceMark rm;
+ AgedRegionData* sorted_regions = NEW_RESOURCE_ARRAY(AgedRegionData, num_regions);
+
+ for (size_t i = 0; i < num_regions; i++) {
+ ShenandoahHeapRegion* const r = heap->get_region(i);
+ if (r->is_empty() || !r->has_live() || !r->is_young() || !r->is_regular()) {
+ // skip over regions that aren't regular young with some live data
+ continue;
+ }
+ if (r->age() >= tenuring_threshold) {
+ if ((r->garbage() < old_garbage_threshold)) {
+ // This tenure-worthy region has too little garbage, so we do not want to expend the copying effort to
+ // reclaim the garbage; instead this region may be eligible for promotion-in-place to the
+ // old generation.
+ HeapWord* tams = ctx->top_at_mark_start(r);
+ HeapWord* original_top = r->top();
+ if (!heap->is_concurrent_old_mark_in_progress() && tams == original_top) {
+ // No allocations from this region have been made during concurrent mark. It meets all the criteria
+ // for in-place-promotion. Though we only need the value of top when we fill the end of the region,
+ // we use this field to indicate that this region should be promoted in place during the evacuation
+ // phase.
+ r->save_top_before_promote();
+
+ size_t remnant_size = r->free() / HeapWordSize;
+ if (remnant_size > ShenandoahHeap::min_fill_size()) {
+ ShenandoahHeap::fill_with_object(original_top, remnant_size);
+ // Fill the remnant memory within this region to assure no allocations prior to promote in place. Otherwise,
+ // newly allocated objects will not be parsable when promote in place tries to register them. Furthermore, any
+ // new allocations would not necessarily be eligible for promotion. This addresses both issues.
+ r->set_top(r->end());
+ promote_in_place_pad += remnant_size * HeapWordSize;
+ } else {
+ // Since the remnant is so small that it cannot be filled, we don't have to worry about any accidental
+ // allocations occurring within this region before the region is promoted in place.
+ }
+ }
+ // Else, we do not promote this region (either in place or by copy) because it has received new allocations.
+
+ // During evacuation, we exclude from promotion regions for which age > tenure threshold, garbage < garbage-threshold,
+ // and get_top_before_promote() != tams
+ } else {
+ // Record this promotion-eligible candidate region. After sorting and selecting the best candidates below,
+ // we may still decide to exclude this promotion-eligible region from the current collection set. If this
+ // happens, we will consider this region as part of the anticipated promotion potential for the next GC
+ // pass; see further below.
+ sorted_regions[candidates]._region = r;
+ sorted_regions[candidates++]._live_data = r->get_live_data_bytes();
+ }
+ } else {
+ // We only evacuate & promote objects from regular regions whose garbage() is above old-garbage-threshold.
+ // Objects in tenure-worthy regions with less garbage are promoted in place. These take a different path to
+ // old-gen. Regions excluded from promotion because their garbage content is too low (causing us to anticipate that
+ // the region would be promoted in place) may be eligible for evacuation promotion by the time promotion takes
+ // place during a subsequent GC pass because more garbage is found within the region between now and then. This
+ // should not happen if we are properly adapting the tenure age. The theory behind adaptive tenuring threshold
+ // is to choose the youngest age that demonstrates no "significant" further loss of population since the previous
+ // age. If not this, we expect the tenure age to demonstrate linear population decay for at least two population
+ // samples, whereas we expect to observe exponential population decay for ages younger than the tenure age.
+ //
+ // In the case that certain regions which were anticipated to be promoted in place need to be promoted by
+ // evacuation, it may be the case that there is not sufficient reserve within old-gen to hold evacuation of
+ // these regions. The likely outcome is that these regions will not be selected for evacuation or promotion
+ // in the current cycle and we will anticipate that they will be promoted in the next cycle. This will cause
+ // us to reserve more old-gen memory so that these objects can be promoted in the subsequent cycle.
+ if (heap->is_aging_cycle() && (r->age() + 1 == tenuring_threshold)) {
+ if (r->garbage() >= old_garbage_threshold) {
+ promo_potential += r->get_live_data_bytes();
+ }
+ }
+ }
+ // Note that we keep going even if one region is excluded from selection.
+ // Subsequent regions may be selected if they have smaller live data.
+ }
+ // Sort in increasing order according to live data bytes. Note that candidates represents the number of regions
+ // that qualify to be promoted by evacuation.
+ if (candidates > 0) {
+ size_t selected_regions = 0;
+ size_t selected_live = 0;
+ QuickSort::sort(sorted_regions, candidates, compare_by_aged_live);
+ for (size_t i = 0; i < candidates; i++) {
+ ShenandoahHeapRegion* const region = sorted_regions[i]._region;
+ size_t region_live_data = sorted_regions[i]._live_data;
+ size_t promotion_need = (size_t) (region_live_data * ShenandoahPromoEvacWaste);
+ if (old_consumed + promotion_need <= old_available) {
+ old_consumed += promotion_need;
+ candidate_regions_for_promotion_by_copy[region->index()] = true;
+ selected_regions++;
+ selected_live += region_live_data;
+ } else {
+ // We rejected this promotable region from the collection set because we had no room to hold its copy.
+ // Add this region to promo potential for next GC.
+ promo_potential += region_live_data;
+ assert(!candidate_regions_for_promotion_by_copy[region->index()], "Shouldn't be selected");
+ }
+ // We keep going even if one region is excluded from selection because we need to accumulate all eligible
+ // regions that are not preselected into promo_potential
+ }
+ log_debug(gc)("Preselected " SIZE_FORMAT " regions containing " SIZE_FORMAT " live bytes,"
+ " consuming: " SIZE_FORMAT " of budgeted: " SIZE_FORMAT,
+ selected_regions, selected_live, old_consumed, old_available);
+ }
+
+ heap->old_generation()->set_pad_for_promote_in_place(promote_in_place_pad);
+ heap->old_generation()->set_promotion_potential(promo_potential);
+ return old_consumed;
+}
+
+void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) {
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+ ShenandoahCollectionSet* collection_set = heap->collection_set();
+ bool is_generational = heap->mode()->is_generational();
+
+ assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
+ assert(!is_old(), "Only YOUNG and GLOBAL GC perform evacuations");
+ {
+ ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states :
+ ShenandoahPhaseTimings::degen_gc_final_update_region_states);
+ ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context());
+ parallel_heap_region_iterate(&cl);
+
+ if (is_young()) {
+ // We always need to update the watermark for old regions. If there
+ // are mixed collections pending, we also need to synchronize the
+ // pinned status for old regions. Since we are already visiting every
+ // old region here, go ahead and sync the pin status too.
+ ShenandoahFinalMarkUpdateRegionStateClosure old_cl(nullptr);
+ heap->old_generation()->parallel_heap_region_iterate(&old_cl);
+ }
+ }
+
+ // Tally the census counts and compute the adaptive tenuring threshold
+ if (is_generational && ShenandoahGenerationalAdaptiveTenuring && !ShenandoahGenerationalCensusAtEvac) {
+ // Objects above TAMS weren't included in the age census. Since they were all
+ // allocated in this cycle they belong in the age 0 cohort. We walk over all
+ // young regions and sum the volume of objects between TAMS and top.
+ ShenandoahUpdateCensusZeroCohortClosure age0_cl(complete_marking_context());
+ heap->young_generation()->heap_region_iterate(&age0_cl);
+ size_t age0_pop = age0_cl.get_age0_population();
+
+ // Update the global census, including the missed age 0 cohort above,
+ // along with the census done during marking, and compute the tenuring threshold.
+ ShenandoahAgeCensus* census = ShenandoahGenerationalHeap::heap()->age_census();
+ census->update_census(age0_pop);
+#ifndef PRODUCT
+ size_t total_pop = age0_cl.get_total_population();
+ size_t total_census = census->get_total();
+ // Usually total_pop > total_census, but not by too much.
+ // We use integer division so anything up to just less than 2 is considered
+ // reasonable, and the "+1" is to avoid divide-by-zero.
+ assert((total_pop+1)/(total_census+1) == 1, "Extreme divergence: "
+ SIZE_FORMAT "/" SIZE_FORMAT, total_pop, total_census);
+#endif
+ }
+
+ {
+ ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset :
+ ShenandoahPhaseTimings::degen_gc_choose_cset);
+
+ collection_set->clear();
+ ShenandoahHeapLocker locker(heap->lock());
+ if (is_generational) {
+ // Seed the collection set with resource area-allocated
+ // preselected regions, which are removed when we exit this scope.
+ ShenandoahCollectionSetPreselector preselector(collection_set, heap->num_regions());
+
+ // Find the amount that will be promoted, regions that will be promoted in
+ // place, and preselect older regions that will be promoted by evacuation.
+ compute_evacuation_budgets(heap);
+
+ // Choose the collection set, including the regions preselected above for
+ // promotion into the old generation.
+ _heuristics->choose_collection_set(collection_set);
+ if (!collection_set->is_empty()) {
+ // only make use of evacuation budgets when we are evacuating
+ adjust_evacuation_budgets(heap, collection_set);
+ }
+
+ if (is_global()) {
+ // We have just chosen a collection set for a global cycle. The mark bitmap covering old regions is complete, so
+ // the remembered set scan can use that to avoid walking into garbage. When the next old mark begins, we will
+ // use the mark bitmap to make the old regions parsable by coalescing and filling any unmarked objects. Thus,
+ // we prepare for old collections by remembering which regions are old at this time. Note that any objects
+ // promoted into old regions will be above TAMS, and so will be considered marked. However, free regions that
+ // become old after this point will not be covered correctly by the mark bitmap, so we must be careful not to
+ // coalesce those regions. Only the old regions which are not part of the collection set at this point are
+ // eligible for coalescing. As implemented now, this has the side effect of possibly initiating mixed-evacuations
+ // after a global cycle for old regions that were not included in this collection set.
+ heap->old_generation()->prepare_for_mixed_collections_after_global_gc();
+ }
+ } else {
+ _heuristics->choose_collection_set(collection_set);
+ }
+ }
+
+
+ {
+ ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset :
+ ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset);
+ ShenandoahHeapLocker locker(heap->lock());
+ size_t young_cset_regions, old_cset_regions;
+
+ // We are preparing for evacuation. At this time, we ignore cset region tallies.
+ size_t first_old, last_old, num_old;
+ heap->free_set()->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old);
+ // Free set construction uses reserve quantities, because they are known to be valid here
+ heap->free_set()->finish_rebuild(young_cset_regions, old_cset_regions, num_old, true);
+ }
+}
+
+bool ShenandoahGeneration::is_bitmap_clear() {
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+ ShenandoahMarkingContext* context = heap->marking_context();
+ const size_t num_regions = heap->num_regions();
+ for (size_t idx = 0; idx < num_regions; idx++) {
+ ShenandoahHeapRegion* r = heap->get_region(idx);
+ if (contains(r) && r->is_affiliated()) {
+ if (heap->is_bitmap_slice_committed(r) && (context->top_at_mark_start(r) > r->bottom()) &&
+ !context->is_bitmap_range_within_region_clear(r->bottom(), r->end())) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+bool ShenandoahGeneration::is_mark_complete() {
+ return _is_marking_complete.is_set();
+}
+
+void ShenandoahGeneration::set_mark_complete() {
+ _is_marking_complete.set();
+}
+
+void ShenandoahGeneration::set_mark_incomplete() {
+ _is_marking_complete.unset();
+}
+
+ShenandoahMarkingContext* ShenandoahGeneration::complete_marking_context() {
+ assert(is_mark_complete(), "Marking must be completed.");
+ return ShenandoahHeap::heap()->marking_context();
+}
+
+void ShenandoahGeneration::cancel_marking() {
+ log_info(gc)("Cancel marking: %s", name());
+ if (is_concurrent_mark_in_progress()) {
+ set_mark_incomplete();
+ }
+ _task_queues->clear();
+ ref_processor()->abandon_partial_discovery();
+ set_concurrent_mark_in_progress(false);
+}
+
+ShenandoahGeneration::ShenandoahGeneration(ShenandoahGenerationType type,
+ uint max_workers,
+ size_t max_capacity,
+ size_t soft_max_capacity) :
+ _type(type),
+ _task_queues(new ShenandoahObjToScanQueueSet(max_workers)),
+ _ref_processor(new ShenandoahReferenceProcessor(MAX2(max_workers, 1U))),
+ _affiliated_region_count(0), _humongous_waste(0), _evacuation_reserve(0),
+ _used(0), _bytes_allocated_since_gc_start(0),
+ _max_capacity(max_capacity), _soft_max_capacity(soft_max_capacity),
+ _heuristics(nullptr)
+{
+ _is_marking_complete.set();
+ assert(max_workers > 0, "At least one queue");
+ for (uint i = 0; i < max_workers; ++i) {
+ ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
+ _task_queues->register_queue(i, task_queue);
+ }
+}
+
+ShenandoahGeneration::~ShenandoahGeneration() {
+ for (uint i = 0; i < _task_queues->size(); ++i) {
+ ShenandoahObjToScanQueue* q = _task_queues->queue(i);
+ delete q;
+ }
+ delete _task_queues;
+}
+
+void ShenandoahGeneration::reserve_task_queues(uint workers) {
+ _task_queues->reserve(workers);
+}
+
+ShenandoahObjToScanQueueSet* ShenandoahGeneration::old_gen_task_queues() const {
+ return nullptr;
+}
+
+void ShenandoahGeneration::scan_remembered_set(bool is_concurrent) {
+ assert(is_young(), "Should only scan remembered set for young generation.");
+
+ ShenandoahGenerationalHeap* const heap = ShenandoahGenerationalHeap::heap();
+ uint nworkers = heap->workers()->active_workers();
+ reserve_task_queues(nworkers);
+
+ ShenandoahReferenceProcessor* rp = ref_processor();
+ ShenandoahRegionChunkIterator work_list(nworkers);
+ ShenandoahScanRememberedTask task(task_queues(), old_gen_task_queues(), rp, &work_list, is_concurrent);
+ heap->assert_gc_workers(nworkers);
+ heap->workers()->run_task(&task);
+ if (ShenandoahEnableCardStats) {
+ ShenandoahScanRemembered* scanner = heap->old_generation()->card_scan();
+ assert(scanner != nullptr, "Not generational");
+ scanner->log_card_stats(nworkers, CARD_STAT_SCAN_RS);
+ }
+}
+
+size_t ShenandoahGeneration::increment_affiliated_region_count() {
+ shenandoah_assert_heaplocked_or_safepoint();
+ // During full gc, multiple GC worker threads may change region affiliations without a lock. No lock is enforced
+ // on read and write of _affiliated_region_count. At the end of full gc, a single thread overwrites the count with
+ // a coherent value.
+ _affiliated_region_count++;
+ return _affiliated_region_count;
+}
+
+size_t ShenandoahGeneration::decrement_affiliated_region_count() {
+ shenandoah_assert_heaplocked_or_safepoint();
+ // During full gc, multiple GC worker threads may change region affiliations without a lock. No lock is enforced
+ // on read and write of _affiliated_region_count. At the end of full gc, a single thread overwrites the count with
+ // a coherent value.
+ _affiliated_region_count--;
+ assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
+ (_used + _humongous_waste <= _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()),
+ "used + humongous cannot exceed regions");
+ return _affiliated_region_count;
+}
+
+size_t ShenandoahGeneration::increase_affiliated_region_count(size_t delta) {
+ shenandoah_assert_heaplocked_or_safepoint();
+ _affiliated_region_count += delta;
+ return _affiliated_region_count;
+}
+
+size_t ShenandoahGeneration::decrease_affiliated_region_count(size_t delta) {
+ shenandoah_assert_heaplocked_or_safepoint();
+ assert(_affiliated_region_count >= delta, "Affiliated region count cannot be negative");
+
+ _affiliated_region_count -= delta;
+ assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
+ (_used + _humongous_waste <= _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()),
+ "used + humongous cannot exceed regions");
+ return _affiliated_region_count;
+}
+
+void ShenandoahGeneration::establish_usage(size_t num_regions, size_t num_bytes, size_t humongous_waste) {
+ assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
+ _affiliated_region_count = num_regions;
+ _used = num_bytes;
+ _humongous_waste = humongous_waste;
+}
+
+void ShenandoahGeneration::increase_used(size_t bytes) {
+ Atomic::add(&_used, bytes);
+}
+
+void ShenandoahGeneration::increase_humongous_waste(size_t bytes) {
+ if (bytes > 0) {
+ Atomic::add(&_humongous_waste, bytes);
+ }
+}
+
+void ShenandoahGeneration::decrease_humongous_waste(size_t bytes) {
+ if (bytes > 0) {
+ assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || (_humongous_waste >= bytes),
+ "Waste (" SIZE_FORMAT ") cannot be negative (after subtracting " SIZE_FORMAT ")", _humongous_waste, bytes);
+ Atomic::sub(&_humongous_waste, bytes);
+ }
+}
+
+void ShenandoahGeneration::decrease_used(size_t bytes) {
+ assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
+ (_used >= bytes), "cannot reduce bytes used by generation below zero");
+ Atomic::sub(&_used, bytes);
+}
+
+size_t ShenandoahGeneration::used_regions() const {
+ return _affiliated_region_count;
+}
+
+size_t ShenandoahGeneration::free_unaffiliated_regions() const {
+ size_t result = max_capacity() / ShenandoahHeapRegion::region_size_bytes();
+ if (_affiliated_region_count > result) {
+ result = 0;
+ } else {
+ result -= _affiliated_region_count;
+ }
+ return result;
+}
+
+size_t ShenandoahGeneration::used_regions_size() const {
+ return _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes();
+}
+
+size_t ShenandoahGeneration::available() const {
+ return available(max_capacity());
+}
+
+// For ShenandoahYoungGeneration, Include the young available that may have been reserved for the Collector.
+size_t ShenandoahGeneration::available_with_reserve() const {
+ return available(max_capacity());
+}
+
+size_t ShenandoahGeneration::soft_available() const {
+ return available(soft_max_capacity());
+}
+
+size_t ShenandoahGeneration::available(size_t capacity) const {
+ size_t in_use = used() + get_humongous_waste();
+ return in_use > capacity ? 0 : capacity - in_use;
+}
+
+size_t ShenandoahGeneration::increase_capacity(size_t increment) {
+ shenandoah_assert_heaplocked_or_safepoint();
+
+ // We do not enforce that new capacity >= heap->max_size_for(this). The maximum generation size is treated as a rule of thumb
+ // which may be violated during certain transitions, such as when we are forcing transfers for the purpose of promoting regions
+ // in place.
+ assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
+ (_max_capacity + increment <= ShenandoahHeap::heap()->max_capacity()), "Generation cannot be larger than heap size");
+ assert(increment % ShenandoahHeapRegion::region_size_bytes() == 0, "Generation capacity must be multiple of region size");
+ _max_capacity += increment;
+
+ // This detects arithmetic wraparound on _used
+ assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
+ (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() >= _used),
+ "Affiliated regions must hold more than what is currently used");
+ return _max_capacity;
+}
+
+size_t ShenandoahGeneration::set_capacity(size_t byte_size) {
+ shenandoah_assert_heaplocked_or_safepoint();
+ _max_capacity = byte_size;
+ return _max_capacity;
+}
+
+size_t ShenandoahGeneration::decrease_capacity(size_t decrement) {
+ shenandoah_assert_heaplocked_or_safepoint();
+
+ // We do not enforce that new capacity >= heap->min_size_for(this). The minimum generation size is treated as a rule of thumb
+ // which may be violated during certain transitions, such as when we are forcing transfers for the purpose of promoting regions
+ // in place.
+ assert(decrement % ShenandoahHeapRegion::region_size_bytes() == 0, "Generation capacity must be multiple of region size");
+ assert(_max_capacity >= decrement, "Generation capacity cannot be negative");
+
+ _max_capacity -= decrement;
+
+ // This detects arithmetic wraparound on _used
+ assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
+ (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() >= _used),
+ "Affiliated regions must hold more than what is currently used");
+ assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
+ (_used <= _max_capacity), "Cannot use more than capacity");
+ assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
+ (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() <= _max_capacity),
+ "Cannot use more than capacity");
+ return _max_capacity;
+}
+
+void ShenandoahGeneration::record_success_concurrent(bool abbreviated) {
+ heuristics()->record_success_concurrent();
+ ShenandoahHeap::heap()->shenandoah_policy()->record_success_concurrent(is_young(), abbreviated);
+}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp
new file mode 100644
index 00000000000..f43c295dff5
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp
@@ -0,0 +1,244 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHGENERATION_HPP
+#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHGENERATION_HPP
+
+#include "memory/allocation.hpp"
+#include "gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp"
+#include "gc/shenandoah/shenandoahAffiliation.hpp"
+#include "gc/shenandoah/shenandoahGenerationType.hpp"
+#include "gc/shenandoah/shenandoahLock.hpp"
+#include "gc/shenandoah/shenandoahMarkingContext.hpp"
+
+class ShenandoahCollectionSet;
+class ShenandoahHeap;
+class ShenandoahHeapRegion;
+class ShenandoahHeapRegionClosure;
+class ShenandoahHeuristics;
+class ShenandoahMode;
+class ShenandoahReferenceProcessor;
+
+
+class ShenandoahGeneration : public CHeapObj, public ShenandoahSpaceInfo {
+ friend class VMStructs;
+private:
+ ShenandoahGenerationType const _type;
+
+ // Marking task queues and completeness
+ ShenandoahObjToScanQueueSet* _task_queues;
+ ShenandoahSharedFlag _is_marking_complete;
+
+ ShenandoahReferenceProcessor* const _ref_processor;
+
+ size_t _affiliated_region_count;
+
+ // How much free memory is left in the last region of humongous objects.
+ // This is _not_ included in used, but it _is_ deducted from available,
+ // which gives the heuristics a more accurate view of how much memory remains
+ // for allocation. This figure is also included the heap status logging.
+ // The units are bytes. The value is only changed on a safepoint or under the
+ // heap lock.
+ size_t _humongous_waste;
+
+ // Bytes reserved within this generation to hold evacuated objects from the collection set
+ size_t _evacuation_reserve;
+
+protected:
+ // Usage
+
+ volatile size_t _used;
+ volatile size_t _bytes_allocated_since_gc_start;
+ size_t _max_capacity;
+ size_t _soft_max_capacity;
+
+ ShenandoahHeuristics* _heuristics;
+
+private:
+ // Compute evacuation budgets prior to choosing collection set.
+ void compute_evacuation_budgets(ShenandoahHeap* heap);
+
+ // Adjust evacuation budgets after choosing collection set.
+ void adjust_evacuation_budgets(ShenandoahHeap* heap,
+ ShenandoahCollectionSet* collection_set);
+
+ // Preselect for possible inclusion into the collection set exactly the most
+ // garbage-dense regions, including those that satisfy criteria 1 & 2 below,
+ // and whose live bytes will fit within old_available budget:
+ // Criterion 1. region age >= tenuring threshold
+ // Criterion 2. region garbage percentage > ShenandoahOldGarbageThreshold
+ //
+ // Identifies regions eligible for promotion in place,
+ // being those of at least tenuring_threshold age that have lower garbage
+ // density.
+ //
+ // Updates promotion_potential and pad_for_promote_in_place fields
+ // of the heap. Returns bytes of live object memory in the preselected
+ // regions, which are marked in the preselected_regions() indicator
+ // array of the heap's collection set, which should be initialized
+ // to false.
+ size_t select_aged_regions(size_t old_available);
+
+ size_t available(size_t capacity) const;
+
+ public:
+ ShenandoahGeneration(ShenandoahGenerationType type,
+ uint max_workers,
+ size_t max_capacity,
+ size_t soft_max_capacity);
+ ~ShenandoahGeneration();
+
+ bool is_young() const { return _type == YOUNG; }
+ bool is_old() const { return _type == OLD; }
+ bool is_global() const { return _type == GLOBAL || _type == NON_GEN; }
+
+ // see description in field declaration
+ void set_evacuation_reserve(size_t new_val);
+ size_t get_evacuation_reserve() const;
+ void augment_evacuation_reserve(size_t increment);
+
+ inline ShenandoahGenerationType type() const { return _type; }
+
+ virtual ShenandoahHeuristics* heuristics() const { return _heuristics; }
+
+ ShenandoahReferenceProcessor* ref_processor() { return _ref_processor; }
+
+ virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode);
+
+ size_t soft_max_capacity() const override { return _soft_max_capacity; }
+ size_t max_capacity() const override { return _max_capacity; }
+ virtual size_t used_regions() const;
+ virtual size_t used_regions_size() const;
+ virtual size_t free_unaffiliated_regions() const;
+ size_t used() const override { return _used; }
+ size_t available() const override;
+ size_t available_with_reserve() const;
+ size_t used_including_humongous_waste() const {
+ return used() + get_humongous_waste();
+ }
+
+ // Returns the memory available based on the _soft_ max heap capacity (soft_max_heap - used).
+ // The soft max heap size may be adjusted lower than the max heap size to cause the trigger
+ // to believe it has less memory available than is _really_ available. Lowering the soft
+ // max heap size will cause the adaptive heuristic to run more frequent cycles.
+ size_t soft_available() const override;
+
+ size_t bytes_allocated_since_gc_start() const override;
+ void reset_bytes_allocated_since_gc_start();
+ void increase_allocated(size_t bytes);
+
+ // These methods change the capacity of the generation by adding or subtracting the given number of bytes from the current
+ // capacity, returning the capacity of the generation following the change.
+ size_t increase_capacity(size_t increment);
+ size_t decrease_capacity(size_t decrement);
+
+ // Set the capacity of the generation, returning the value set
+ size_t set_capacity(size_t byte_size);
+
+ void log_status(const char* msg) const;
+
+ // Used directly by FullGC
+ void reset_mark_bitmap();
+
+ // Used by concurrent and degenerated GC to reset remembered set.
+ void swap_remembered_set();
+
+ // Update the read cards with the state of the write table (write table is not cleared).
+ void merge_write_table();
+
+ // Called before init mark, expected to prepare regions for marking.
+ virtual void prepare_gc();
+
+ // Called during final mark, chooses collection set, rebuilds free set.
+ virtual void prepare_regions_and_collection_set(bool concurrent);
+
+ // Cancel marking (used by Full collect and when cancelling cycle).
+ virtual void cancel_marking();
+
+ virtual bool contains(ShenandoahAffiliation affiliation) const = 0;
+
+ // Return true if this region is affiliated with this generation.
+ virtual bool contains(ShenandoahHeapRegion* region) const = 0;
+
+ // Return true if this object is affiliated with this generation.
+ virtual bool contains(oop obj) const = 0;
+
+ // Apply closure to all regions affiliated with this generation.
+ virtual void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) = 0;
+
+ // Apply closure to all regions affiliated with this generation (include free regions);
+ virtual void parallel_heap_region_iterate_free(ShenandoahHeapRegionClosure* cl);
+
+ // Apply closure to all regions affiliated with this generation (single threaded).
+ virtual void heap_region_iterate(ShenandoahHeapRegionClosure* cl) = 0;
+
+ // This is public to support cancellation of marking when a Full cycle is started.
+ virtual void set_concurrent_mark_in_progress(bool in_progress) = 0;
+
+ // Check the bitmap only for regions belong to this generation.
+ bool is_bitmap_clear();
+
+ // We need to track the status of marking for different generations.
+ bool is_mark_complete();
+ virtual void set_mark_complete();
+ virtual void set_mark_incomplete();
+
+ ShenandoahMarkingContext* complete_marking_context();
+
+ // Task queues
+ ShenandoahObjToScanQueueSet* task_queues() const { return _task_queues; }
+ virtual void reserve_task_queues(uint workers);
+ virtual ShenandoahObjToScanQueueSet* old_gen_task_queues() const;
+
+ // Scan remembered set at start of concurrent young-gen marking.
+ void scan_remembered_set(bool is_concurrent);
+
+ // Return the updated value of affiliated_region_count
+ size_t increment_affiliated_region_count();
+
+ // Return the updated value of affiliated_region_count
+ size_t decrement_affiliated_region_count();
+
+ // Return the updated value of affiliated_region_count
+ size_t increase_affiliated_region_count(size_t delta);
+
+ // Return the updated value of affiliated_region_count
+ size_t decrease_affiliated_region_count(size_t delta);
+
+ void establish_usage(size_t num_regions, size_t num_bytes, size_t humongous_waste);
+
+ void increase_used(size_t bytes);
+ void decrease_used(size_t bytes);
+
+ void increase_humongous_waste(size_t bytes);
+ void decrease_humongous_waste(size_t bytes);
+ size_t get_humongous_waste() const { return _humongous_waste; }
+
+ virtual bool is_concurrent_mark_in_progress() = 0;
+ void confirm_heuristics_mode();
+
+ virtual void record_success_concurrent(bool abbreviated);
+};
+
+#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHGENERATION_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.cpp
new file mode 100644
index 00000000000..dfbc6b673ff
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.cpp
@@ -0,0 +1,209 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/shenandoah/shenandoahGeneration.hpp"
+#include "gc/shenandoah/shenandoahGenerationSizer.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
+#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "logging/log.hpp"
+#include "runtime/globals_extension.hpp"
+
+
+ShenandoahGenerationSizer::ShenandoahGenerationSizer()
+ : _sizer_kind(SizerDefaults),
+ _min_desired_young_regions(0),
+ _max_desired_young_regions(0) {
+
+ if (FLAG_IS_CMDLINE(NewRatio)) {
+ if (FLAG_IS_CMDLINE(NewSize) || FLAG_IS_CMDLINE(MaxNewSize)) {
+ log_warning(gc, ergo)("-XX:NewSize and -XX:MaxNewSize override -XX:NewRatio");
+ } else {
+ _sizer_kind = SizerNewRatio;
+ return;
+ }
+ }
+
+ if (NewSize > MaxNewSize) {
+ if (FLAG_IS_CMDLINE(MaxNewSize)) {
+ log_warning(gc, ergo)("NewSize (" SIZE_FORMAT "k) is greater than the MaxNewSize (" SIZE_FORMAT "k). "
+ "A new max generation size of " SIZE_FORMAT "k will be used.",
+ NewSize/K, MaxNewSize/K, NewSize/K);
+ }
+ FLAG_SET_ERGO(MaxNewSize, NewSize);
+ }
+
+ if (FLAG_IS_CMDLINE(NewSize)) {
+ _min_desired_young_regions = MAX2(uint(NewSize / ShenandoahHeapRegion::region_size_bytes()), 1U);
+ if (FLAG_IS_CMDLINE(MaxNewSize)) {
+ _max_desired_young_regions = MAX2(uint(MaxNewSize / ShenandoahHeapRegion::region_size_bytes()), 1U);
+ _sizer_kind = SizerMaxAndNewSize;
+ } else {
+ _sizer_kind = SizerNewSizeOnly;
+ }
+ } else if (FLAG_IS_CMDLINE(MaxNewSize)) {
+ _max_desired_young_regions = MAX2(uint(MaxNewSize / ShenandoahHeapRegion::region_size_bytes()), 1U);
+ _sizer_kind = SizerMaxNewSizeOnly;
+ }
+}
+
+size_t ShenandoahGenerationSizer::calculate_min_young_regions(size_t heap_region_count) {
+ size_t min_young_regions = (heap_region_count * ShenandoahMinYoungPercentage) / 100;
+ return MAX2(min_young_regions, (size_t) 1U);
+}
+
+size_t ShenandoahGenerationSizer::calculate_max_young_regions(size_t heap_region_count) {
+ size_t max_young_regions = (heap_region_count * ShenandoahMaxYoungPercentage) / 100;
+ return MAX2(max_young_regions, (size_t) 1U);
+}
+
+void ShenandoahGenerationSizer::recalculate_min_max_young_length(size_t heap_region_count) {
+ assert(heap_region_count > 0, "Heap must be initialized");
+
+ switch (_sizer_kind) {
+ case SizerDefaults:
+ _min_desired_young_regions = calculate_min_young_regions(heap_region_count);
+ _max_desired_young_regions = calculate_max_young_regions(heap_region_count);
+ break;
+ case SizerNewSizeOnly:
+ _max_desired_young_regions = calculate_max_young_regions(heap_region_count);
+ _max_desired_young_regions = MAX2(_min_desired_young_regions, _max_desired_young_regions);
+ break;
+ case SizerMaxNewSizeOnly:
+ _min_desired_young_regions = calculate_min_young_regions(heap_region_count);
+ _min_desired_young_regions = MIN2(_min_desired_young_regions, _max_desired_young_regions);
+ break;
+ case SizerMaxAndNewSize:
+ // Do nothing. Values set on the command line, don't update them at runtime.
+ break;
+ case SizerNewRatio:
+ _min_desired_young_regions = MAX2(uint(heap_region_count / (NewRatio + 1)), 1U);
+ _max_desired_young_regions = _min_desired_young_regions;
+ break;
+ default:
+ ShouldNotReachHere();
+ }
+
+ assert(_min_desired_young_regions <= _max_desired_young_regions, "Invalid min/max young gen size values");
+}
+
+void ShenandoahGenerationSizer::heap_size_changed(size_t heap_size) {
+ recalculate_min_max_young_length(heap_size / ShenandoahHeapRegion::region_size_bytes());
+}
+
+bool ShenandoahGenerationSizer::transfer_regions(ShenandoahGeneration* src, ShenandoahGeneration* dst, size_t regions) const {
+ const size_t bytes_to_transfer = regions * ShenandoahHeapRegion::region_size_bytes();
+
+ if (src->free_unaffiliated_regions() < regions) {
+ // Source does not have enough free regions for this transfer. The caller should have
+ // already capped the transfer based on available unaffiliated regions.
+ return false;
+ }
+
+ if (dst->max_capacity() + bytes_to_transfer > max_size_for(dst)) {
+ // This transfer would cause the destination generation to grow above its configured maximum size.
+ return false;
+ }
+
+ if (src->max_capacity() - bytes_to_transfer < min_size_for(src)) {
+ // This transfer would cause the source generation to shrink below its configured minimum size.
+ return false;
+ }
+
+ src->decrease_capacity(bytes_to_transfer);
+ dst->increase_capacity(bytes_to_transfer);
+ const size_t new_size = dst->max_capacity();
+ log_info(gc, ergo)("Transfer " SIZE_FORMAT " region(s) from %s to %s, yielding increased size: " PROPERFMT,
+ regions, src->name(), dst->name(), PROPERFMTARGS(new_size));
+ return true;
+}
+
+
+size_t ShenandoahGenerationSizer::max_size_for(ShenandoahGeneration* generation) const {
+ switch (generation->type()) {
+ case YOUNG:
+ return max_young_size();
+ case OLD:
+ // On the command line, max size of OLD is specified indirectly, by setting a minimum size of young.
+ // OLD is what remains within the heap after YOUNG has been sized.
+ return ShenandoahHeap::heap()->max_capacity() - min_young_size();
+ default:
+ ShouldNotReachHere();
+ return 0;
+ }
+}
+
+size_t ShenandoahGenerationSizer::min_size_for(ShenandoahGeneration* generation) const {
+ switch (generation->type()) {
+ case YOUNG:
+ return min_young_size();
+ case OLD:
+ // On the command line, min size of OLD is specified indirectly, by setting a maximum size of young.
+ // OLD is what remains within the heap after YOUNG has been sized.
+ return ShenandoahHeap::heap()->max_capacity() - max_young_size();
+ default:
+ ShouldNotReachHere();
+ return 0;
+ }
+}
+
+
+// Returns true iff transfer is successful
+bool ShenandoahGenerationSizer::transfer_to_old(size_t regions) const {
+ ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
+ return transfer_regions(heap->young_generation(), heap->old_generation(), regions);
+}
+
+// This is used when promoting humongous or highly utilized regular regions in place. It is not required in this situation
+// that the transferred regions be unaffiliated.
+void ShenandoahGenerationSizer::force_transfer_to_old(size_t regions) const {
+ ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
+ ShenandoahGeneration* old_gen = heap->old_generation();
+ ShenandoahGeneration* young_gen = heap->young_generation();
+ const size_t bytes_to_transfer = regions * ShenandoahHeapRegion::region_size_bytes();
+
+ young_gen->decrease_capacity(bytes_to_transfer);
+ old_gen->increase_capacity(bytes_to_transfer);
+ const size_t new_size = old_gen->max_capacity();
+ log_info(gc, ergo)("Forcing transfer of " SIZE_FORMAT " region(s) from %s to %s, yielding increased size: " PROPERFMT,
+ regions, young_gen->name(), old_gen->name(), PROPERFMTARGS(new_size));
+}
+
+
+bool ShenandoahGenerationSizer::transfer_to_young(size_t regions) const {
+ ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
+ return transfer_regions(heap->old_generation(), heap->young_generation(), regions);
+}
+
+size_t ShenandoahGenerationSizer::min_young_size() const {
+ return min_young_regions() * ShenandoahHeapRegion::region_size_bytes();
+}
+
+size_t ShenandoahGenerationSizer::max_young_size() const {
+ return max_young_regions() * ShenandoahHeapRegion::region_size_bytes();
+}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.hpp
new file mode 100644
index 00000000000..5752422bb77
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.hpp
@@ -0,0 +1,93 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONSIZER_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONSIZER_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+class ShenandoahGeneration;
+class ShenandoahGenerationalHeap;
+
+class ShenandoahGenerationSizer {
+private:
+ enum SizerKind {
+ SizerDefaults,
+ SizerNewSizeOnly,
+ SizerMaxNewSizeOnly,
+ SizerMaxAndNewSize,
+ SizerNewRatio
+ };
+ SizerKind _sizer_kind;
+
+ size_t _min_desired_young_regions;
+ size_t _max_desired_young_regions;
+
+ static size_t calculate_min_young_regions(size_t heap_region_count);
+ static size_t calculate_max_young_regions(size_t heap_region_count);
+
+ // Update the given values for minimum and maximum young gen length in regions
+ // given the number of heap regions depending on the kind of sizing algorithm.
+ void recalculate_min_max_young_length(size_t heap_region_count);
+
+ // This will attempt to transfer regions from the `src` generation to `dst` generation.
+ // If the transfer would violate the configured minimum size for the source or the configured
+ // maximum size of the destination, it will not perform the transfer and will return false.
+ // Returns true if the transfer is performed.
+ bool transfer_regions(ShenandoahGeneration* src, ShenandoahGeneration* dst, size_t regions) const;
+
+ // Return the configured maximum size in bytes for the given generation.
+ size_t max_size_for(ShenandoahGeneration* generation) const;
+
+ // Return the configured minimum size in bytes for the given generation.
+ size_t min_size_for(ShenandoahGeneration* generation) const;
+
+public:
+ ShenandoahGenerationSizer();
+
+ // Calculate the maximum length of the young gen given the number of regions
+ // depending on the sizing algorithm.
+ void heap_size_changed(size_t heap_size);
+
+ // Minimum size of young generation in bytes as multiple of region size.
+ size_t min_young_size() const;
+ size_t min_young_regions() const {
+ return _min_desired_young_regions;
+ }
+
+ // Maximum size of young generation in bytes as multiple of region size.
+ size_t max_young_size() const;
+ size_t max_young_regions() const {
+ return _max_desired_young_regions;
+ }
+
+ // True if transfer succeeds, else false. See transfer_regions.
+ bool transfer_to_young(size_t regions) const;
+ bool transfer_to_old(size_t regions) const;
+
+ // force transfer is used when we promote humongous objects. May violate min/max limits on generation sizes
+ void force_transfer_to_old(size_t regions) const;
+};
+
+#endif //SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONSIZER_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationType.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationType.hpp
index 1132fd5eb4d..cfc2b36cf1d 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationType.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationType.hpp
@@ -26,13 +26,22 @@
#define SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONTYPE_HPP
enum ShenandoahGenerationType {
- NON_GEN // non-generational
+ NON_GEN, // non-generational
+ GLOBAL, // generational: Global
+ YOUNG, // generational: Young
+ OLD // generational: Old
};
inline const char* shenandoah_generation_name(ShenandoahGenerationType mode) {
switch (mode) {
case NON_GEN:
return "Non-Generational";
+ case GLOBAL:
+ return "Global";
+ case OLD:
+ return "Old";
+ case YOUNG:
+ return "Young";
default:
ShouldNotReachHere();
return "Unknown";
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp
new file mode 100644
index 00000000000..ef0fbf671a0
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp
@@ -0,0 +1,841 @@
+/*
+ * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shenandoah/mode/shenandoahMode.hpp"
+#include "gc/shenandoah/shenandoahAsserts.hpp"
+#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
+#include "gc/shenandoah/shenandoahConcurrentGC.hpp"
+#include "gc/shenandoah/shenandoahGenerationalControlThread.hpp"
+#include "gc/shenandoah/shenandoahDegeneratedGC.hpp"
+#include "gc/shenandoah/shenandoahFreeSet.hpp"
+#include "gc/shenandoah/shenandoahFullGC.hpp"
+#include "gc/shenandoah/shenandoahGeneration.hpp"
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
+#include "gc/shenandoah/shenandoahOldGC.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
+#include "gc/shenandoah/shenandoahPacer.inline.hpp"
+#include "gc/shenandoah/shenandoahUtils.hpp"
+#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
+#include "logging/log.hpp"
+#include "memory/metaspaceUtils.hpp"
+#include "memory/metaspaceStats.hpp"
+#include "runtime/atomic.hpp"
+
+ShenandoahGenerationalControlThread::ShenandoahGenerationalControlThread() :
+ ShenandoahController(),
+ _control_lock(Mutex::nosafepoint - 2, "ShenandoahControlGC_lock", true),
+ _regulator_lock(Mutex::nosafepoint - 2, "ShenandoahRegulatorGC_lock", true),
+ _requested_gc_cause(GCCause::_no_gc),
+ _requested_generation(GLOBAL),
+ _degen_point(ShenandoahGC::_degenerated_outside_cycle),
+ _degen_generation(nullptr),
+ _mode(none) {
+ shenandoah_assert_generational();
+ set_name("Shenandoah Control Thread");
+ create_and_start();
+}
+
+void ShenandoahGenerationalControlThread::run_service() {
+ ShenandoahGenerationalHeap* const heap = ShenandoahGenerationalHeap::heap();
+
+ const GCMode default_mode = concurrent_normal;
+ ShenandoahGenerationType generation = GLOBAL;
+
+ double last_shrink_time = os::elapsedTime();
+ uint age_period = 0;
+
+ // Shrink period avoids constantly polling regions for shrinking.
+ // Having a period 10x lower than the delay would mean we hit the
+ // shrinking with lag of less than 1/10-th of true delay.
+ // ShenandoahUncommitDelay is in msecs, but shrink_period is in seconds.
+ const double shrink_period = (double)ShenandoahUncommitDelay / 1000 / 10;
+
+ ShenandoahCollectorPolicy* const policy = heap->shenandoah_policy();
+
+ // Heuristics are notified of allocation failures here and other outcomes
+ // of the cycle. They're also used here to control whether the Nth consecutive
+ // degenerated cycle should be 'promoted' to a full cycle. The decision to
+ // trigger a cycle or not is evaluated on the regulator thread.
+ ShenandoahHeuristics* global_heuristics = heap->global_generation()->heuristics();
+ while (!in_graceful_shutdown() && !should_terminate()) {
+ // Figure out if we have pending requests.
+ const bool alloc_failure_pending = _alloc_failure_gc.is_set();
+ const bool humongous_alloc_failure_pending = _humongous_alloc_failure_gc.is_set();
+
+ GCCause::Cause cause = Atomic::xchg(&_requested_gc_cause, GCCause::_no_gc);
+
+ const bool is_gc_requested = ShenandoahCollectorPolicy::is_requested_gc(cause);
+
+ // This control loop iteration has seen this much allocation.
+ const size_t allocs_seen = reset_allocs_seen();
+
+ // Check if we have seen a new target for soft max heap size.
+ const bool soft_max_changed = heap->check_soft_max_changed();
+
+ // Choose which GC mode to run in. The block below should select a single mode.
+ set_gc_mode(none);
+ ShenandoahGC::ShenandoahDegenPoint degen_point = ShenandoahGC::_degenerated_unset;
+
+ if (alloc_failure_pending) {
+ // Allocation failure takes precedence: we have to deal with it first thing
+ cause = GCCause::_allocation_failure;
+
+ // Consume the degen point, and seed it with default value
+ degen_point = _degen_point;
+ _degen_point = ShenandoahGC::_degenerated_outside_cycle;
+
+ if (degen_point == ShenandoahGC::_degenerated_outside_cycle) {
+ _degen_generation = heap->young_generation();
+ } else {
+ assert(_degen_generation != nullptr, "Need to know which generation to resume");
+ }
+
+ ShenandoahHeuristics* heuristics = _degen_generation->heuristics();
+ generation = _degen_generation->type();
+ bool old_gen_evacuation_failed = heap->old_generation()->clear_failed_evacuation();
+
+ heuristics->log_trigger("Handle Allocation Failure");
+
+ // Do not bother with degenerated cycle if old generation evacuation failed or if humongous allocation failed
+ if (ShenandoahDegeneratedGC && heuristics->should_degenerate_cycle() &&
+ !old_gen_evacuation_failed && !humongous_alloc_failure_pending) {
+ heuristics->record_allocation_failure_gc();
+ policy->record_alloc_failure_to_degenerated(degen_point);
+ set_gc_mode(stw_degenerated);
+ } else {
+ heuristics->record_allocation_failure_gc();
+ policy->record_alloc_failure_to_full();
+ generation = GLOBAL;
+ set_gc_mode(stw_full);
+ }
+ } else if (is_gc_requested) {
+ generation = GLOBAL;
+ global_heuristics->log_trigger("GC request (%s)", GCCause::to_string(cause));
+ global_heuristics->record_requested_gc();
+
+ if (ShenandoahCollectorPolicy::should_run_full_gc(cause)) {
+ set_gc_mode(stw_full);
+ } else {
+ set_gc_mode(default_mode);
+ // Unload and clean up everything
+ heap->set_unload_classes(global_heuristics->can_unload_classes());
+ }
+ } else {
+ // We should only be here if the regulator requested a cycle or if
+ // there is an old generation mark in progress.
+ if (cause == GCCause::_shenandoah_concurrent_gc) {
+ if (_requested_generation == OLD && heap->old_generation()->is_doing_mixed_evacuations()) {
+ // If a request to start an old cycle arrived while an old cycle was running, but _before_
+ // it chose any regions for evacuation we don't want to start a new old cycle. Rather, we want
+ // the heuristic to run a young collection so that we can evacuate some old regions.
+ assert(!heap->is_concurrent_old_mark_in_progress(), "Should not be running mixed collections and concurrent marking");
+ generation = YOUNG;
+ } else {
+ generation = _requested_generation;
+ }
+
+ // preemption was requested or this is a regular cycle
+ set_gc_mode(default_mode);
+
+ // Don't start a new old marking if there is one already in progress
+ if (generation == OLD && heap->is_concurrent_old_mark_in_progress()) {
+ set_gc_mode(servicing_old);
+ }
+
+ if (generation == GLOBAL) {
+ heap->set_unload_classes(global_heuristics->should_unload_classes());
+ } else {
+ heap->set_unload_classes(false);
+ }
+ } else if (heap->is_concurrent_old_mark_in_progress() || heap->is_prepare_for_old_mark_in_progress()) {
+ // Nobody asked us to do anything, but we have an old-generation mark or old-generation preparation for
+ // mixed evacuation in progress, so resume working on that.
+ log_info(gc)("Resume old GC: marking is%s in progress, preparing is%s in progress",
+ heap->is_concurrent_old_mark_in_progress() ? "" : " NOT",
+ heap->is_prepare_for_old_mark_in_progress() ? "" : " NOT");
+
+ cause = GCCause::_shenandoah_concurrent_gc;
+ generation = OLD;
+ set_gc_mode(servicing_old);
+ heap->set_unload_classes(false);
+ }
+ }
+
+ const bool gc_requested = (gc_mode() != none);
+ assert (!gc_requested || cause != GCCause::_no_gc, "GC cause should be set");
+
+ if (gc_requested) {
+ // Blow away all soft references on this cycle, if handling allocation failure,
+ // either implicit or explicit GC request, or we are requested to do so unconditionally.
+ if (generation == GLOBAL && (alloc_failure_pending || is_gc_requested || ShenandoahAlwaysClearSoftRefs)) {
+ heap->soft_ref_policy()->set_should_clear_all_soft_refs(true);
+ }
+
+ // GC is starting, bump the internal ID
+ update_gc_id();
+
+ heap->reset_bytes_allocated_since_gc_start();
+
+ MetaspaceCombinedStats meta_sizes = MetaspaceUtils::get_combined_statistics();
+
+ // If GC was requested, we are sampling the counters even without actual triggers
+ // from allocation machinery. This captures GC phases more accurately.
+ heap->set_forced_counters_update(true);
+
+ // If GC was requested, we better dump freeset data for performance debugging
+ heap->free_set()->log_status_under_lock();
+
+ // In case this is a degenerated cycle, remember whether original cycle was aging.
+ const bool was_aging_cycle = heap->is_aging_cycle();
+ heap->set_aging_cycle(false);
+
+ switch (gc_mode()) {
+ case concurrent_normal: {
+ // At this point:
+ // if (generation == YOUNG), this is a normal YOUNG cycle
+ // if (generation == OLD), this is a bootstrap OLD cycle
+ // if (generation == GLOBAL), this is a GLOBAL cycle triggered by System.gc()
+ // In all three cases, we want to age old objects if this is an aging cycle
+ if (age_period-- == 0) {
+ heap->set_aging_cycle(true);
+ age_period = ShenandoahAgingCyclePeriod - 1;
+ }
+ service_concurrent_normal_cycle(heap, generation, cause);
+ break;
+ }
+ case stw_degenerated: {
+ heap->set_aging_cycle(was_aging_cycle);
+ service_stw_degenerated_cycle(cause, degen_point);
+ break;
+ }
+ case stw_full: {
+ if (age_period-- == 0) {
+ heap->set_aging_cycle(true);
+ age_period = ShenandoahAgingCyclePeriod - 1;
+ }
+ service_stw_full_cycle(cause);
+ break;
+ }
+ case servicing_old: {
+ assert(generation == OLD, "Expected old generation here");
+ GCIdMark gc_id_mark;
+ service_concurrent_old_cycle(heap, cause);
+ break;
+ }
+ default:
+ ShouldNotReachHere();
+ }
+
+ // If this was the requested GC cycle, notify waiters about it
+ if (is_gc_requested) {
+ notify_gc_waiters();
+ }
+
+ // If this was the allocation failure GC cycle, notify waiters about it
+ if (alloc_failure_pending) {
+ notify_alloc_failure_waiters();
+ }
+
+ // Report current free set state at the end of cycle, whether
+ // it is a normal completion, or the abort.
+ heap->free_set()->log_status_under_lock();
+
+ // Notify Universe about new heap usage. This has implications for
+ // global soft refs policy, and we better report it every time heap
+ // usage goes down.
+ heap->update_capacity_and_used_at_gc();
+
+ // Signal that we have completed a visit to all live objects.
+ heap->record_whole_heap_examined_timestamp();
+
+ // Disable forced counters update, and update counters one more time
+ // to capture the state at the end of GC session.
+ heap->handle_force_counters_update();
+ heap->set_forced_counters_update(false);
+
+ // Retract forceful part of soft refs policy
+ heap->soft_ref_policy()->set_should_clear_all_soft_refs(false);
+
+ // Clear metaspace oom flag, if current cycle unloaded classes
+ if (heap->unload_classes()) {
+ global_heuristics->clear_metaspace_oom();
+ }
+
+ process_phase_timings(heap);
+
+ // Print Metaspace change following GC (if logging is enabled).
+ MetaspaceUtils::print_metaspace_change(meta_sizes);
+
+ // GC is over, we are at idle now
+ if (ShenandoahPacing) {
+ heap->pacer()->setup_for_idle();
+ }
+ } else {
+ // Report to pacer that we have seen this many words allocated
+ if (ShenandoahPacing && (allocs_seen > 0)) {
+ heap->pacer()->report_alloc(allocs_seen);
+ }
+ }
+
+ const double current = os::elapsedTime();
+
+ if (ShenandoahUncommit && (is_gc_requested || soft_max_changed || (current - last_shrink_time > shrink_period))) {
+ // Explicit GC tries to uncommit everything down to min capacity.
+ // Soft max change tries to uncommit everything down to target capacity.
+ // Periodic uncommit tries to uncommit suitable regions down to min capacity.
+
+ double shrink_before = (is_gc_requested || soft_max_changed) ?
+ current :
+ current - (ShenandoahUncommitDelay / 1000.0);
+
+ size_t shrink_until = soft_max_changed ?
+ heap->soft_max_capacity() :
+ heap->min_capacity();
+
+ heap->maybe_uncommit(shrink_before, shrink_until);
+ heap->phase_timings()->flush_cycle_to_global();
+ last_shrink_time = current;
+ }
+
+ // Wait for ShenandoahControlIntervalMax unless there was an allocation failure or another request was made mid-cycle.
+ if (!is_alloc_failure_gc() && _requested_gc_cause == GCCause::_no_gc) {
+ // The timed wait is necessary because this thread has a responsibility to send
+ // 'alloc_words' to the pacer when it does not perform a GC.
+ MonitorLocker lock(&_control_lock, Mutex::_no_safepoint_check_flag);
+ lock.wait(ShenandoahControlIntervalMax);
+ }
+ }
+
+ // Wait for the actual stop(), can't leave run_service() earlier.
+ while (!should_terminate()) {
+ os::naked_short_sleep(ShenandoahControlIntervalMin);
+ }
+}
+
+void ShenandoahGenerationalControlThread::process_phase_timings(const ShenandoahGenerationalHeap* heap) {
+ // Commit worker statistics to cycle data
+ heap->phase_timings()->flush_par_workers_to_cycle();
+ if (ShenandoahPacing) {
+ heap->pacer()->flush_stats_to_cycle();
+ }
+
+ ShenandoahEvacuationTracker* evac_tracker = heap->evac_tracker();
+ ShenandoahCycleStats evac_stats = evac_tracker->flush_cycle_to_global();
+
+ // Print GC stats for current cycle
+ {
+ LogTarget(Info, gc, stats) lt;
+ if (lt.is_enabled()) {
+ ResourceMark rm;
+ LogStream ls(lt);
+ heap->phase_timings()->print_cycle_on(&ls);
+ evac_tracker->print_evacuations_on(&ls, &evac_stats.workers,
+ &evac_stats.mutators);
+ if (ShenandoahPacing) {
+ heap->pacer()->print_cycle_on(&ls);
+ }
+ }
+ }
+
+ // Commit statistics to globals
+ heap->phase_timings()->flush_cycle_to_global();
+}
+
+// Young and old concurrent cycles are initiated by the regulator. Implicit
+// and explicit GC requests are handled by the controller thread and always
+// run a global cycle (which is concurrent by default, but may be overridden
+// by command line options). Old cycles always degenerate to a global cycle.
+// Young cycles are degenerated to complete the young cycle. Young
+// and old degen may upgrade to Full GC. Full GC may also be
+// triggered directly by a System.gc() invocation.
+//
+//
+// +-----+ Idle +-----+-----------+---------------------+
+// | + | | |
+// | | | | |
+// | | v | |
+// | | Bootstrap Old +-- | ------------+ |
+// | | + | | |
+// | | | | | |
+// | v v v v |
+// | Resume Old <----------+ Young +--> Young Degen |
+// | + + ^ + + |
+// v | | | | | |
+// Global <-+ | +----------------------------+ | |
+// + | | |
+// | v v |
+// +---> Global Degen +--------------------> Full <----+
+//
+void ShenandoahGenerationalControlThread::service_concurrent_normal_cycle(ShenandoahGenerationalHeap* heap,
+ const ShenandoahGenerationType generation,
+ GCCause::Cause cause) {
+ GCIdMark gc_id_mark;
+ switch (generation) {
+ case YOUNG: {
+ // Run a young cycle. This might or might not, have interrupted an ongoing
+ // concurrent mark in the old generation. We need to think about promotions
+ // in this case. Promoted objects should be above the TAMS in the old regions
+ // they end up in, but we have to be sure we don't promote into any regions
+ // that are in the cset.
+ log_info(gc, ergo)("Start GC cycle (Young)");
+ service_concurrent_cycle(heap->young_generation(), cause, false);
+ break;
+ }
+ case OLD: {
+ log_info(gc, ergo)("Start GC cycle (Old)");
+ service_concurrent_old_cycle(heap, cause);
+ break;
+ }
+ case GLOBAL: {
+ log_info(gc, ergo)("Start GC cycle (Global)");
+ service_concurrent_cycle(heap->global_generation(), cause, false);
+ break;
+ }
+ default:
+ ShouldNotReachHere();
+ }
+}
+
+void ShenandoahGenerationalControlThread::service_concurrent_old_cycle(ShenandoahGenerationalHeap* heap, GCCause::Cause &cause) {
+ ShenandoahOldGeneration* old_generation = heap->old_generation();
+ ShenandoahYoungGeneration* young_generation = heap->young_generation();
+ ShenandoahOldGeneration::State original_state = old_generation->state();
+
+ TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
+
+ switch (original_state) {
+ case ShenandoahOldGeneration::FILLING: {
+ ShenandoahGCSession session(cause, old_generation);
+ _allow_old_preemption.set();
+ old_generation->entry_coalesce_and_fill();
+ _allow_old_preemption.unset();
+
+ // Before bootstrapping begins, we must acknowledge any cancellation request.
+ // If the gc has not been cancelled, this does nothing. If it has been cancelled,
+ // this will clear the cancellation request and exit before starting the bootstrap
+ // phase. This will allow the young GC cycle to proceed normally. If we do not
+ // acknowledge the cancellation request, the subsequent young cycle will observe
+ // the request and essentially cancel itself.
+ if (check_cancellation_or_degen(ShenandoahGC::_degenerated_outside_cycle)) {
+ log_info(gc)("Preparation for old generation cycle was cancelled");
+ return;
+ }
+
+ // Coalescing threads completed and nothing was cancelled. it is safe to transition from this state.
+ old_generation->transition_to(ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP);
+ return;
+ }
+ case ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP:
+ old_generation->transition_to(ShenandoahOldGeneration::BOOTSTRAPPING);
+ case ShenandoahOldGeneration::BOOTSTRAPPING: {
+ // Configure the young generation's concurrent mark to put objects in
+ // old regions into the concurrent mark queues associated with the old
+ // generation. The young cycle will run as normal except that rather than
+ // ignore old references it will mark and enqueue them in the old concurrent
+ // task queues but it will not traverse them.
+ set_gc_mode(bootstrapping_old);
+ young_generation->set_old_gen_task_queues(old_generation->task_queues());
+ ShenandoahGCSession session(cause, young_generation);
+ service_concurrent_cycle(heap, young_generation, cause, true);
+ process_phase_timings(heap);
+ if (heap->cancelled_gc()) {
+ // Young generation bootstrap cycle has failed. Concurrent mark for old generation
+ // is going to resume after degenerated bootstrap cycle completes.
+ log_info(gc)("Bootstrap cycle for old generation was cancelled");
+ return;
+ }
+
+ // Reset the degenerated point. Normally this would happen at the top
+ // of the control loop, but here we have just completed a young cycle
+ // which has bootstrapped the old concurrent marking.
+ _degen_point = ShenandoahGC::_degenerated_outside_cycle;
+
+ // From here we will 'resume' the old concurrent mark. This will skip reset
+ // and init mark for the concurrent mark. All of that work will have been
+ // done by the bootstrapping young cycle.
+ set_gc_mode(servicing_old);
+ old_generation->transition_to(ShenandoahOldGeneration::MARKING);
+ }
+ case ShenandoahOldGeneration::MARKING: {
+ ShenandoahGCSession session(cause, old_generation);
+ bool marking_complete = resume_concurrent_old_cycle(old_generation, cause);
+ if (marking_complete) {
+ assert(old_generation->state() != ShenandoahOldGeneration::MARKING, "Should not still be marking");
+ if (original_state == ShenandoahOldGeneration::MARKING) {
+ heap->mmu_tracker()->record_old_marking_increment(true);
+ heap->log_heap_status("At end of Concurrent Old Marking finishing increment");
+ }
+ } else if (original_state == ShenandoahOldGeneration::MARKING) {
+ heap->mmu_tracker()->record_old_marking_increment(false);
+ heap->log_heap_status("At end of Concurrent Old Marking increment");
+ }
+ break;
+ }
+ default:
+ fatal("Unexpected state for old GC: %s", ShenandoahOldGeneration::state_name(old_generation->state()));
+ }
+}
+
+bool ShenandoahGenerationalControlThread::resume_concurrent_old_cycle(ShenandoahOldGeneration* generation, GCCause::Cause cause) {
+ assert(ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress(), "Old mark should be in progress");
+ log_debug(gc)("Resuming old generation with " UINT32_FORMAT " marking tasks queued", generation->task_queues()->tasks());
+
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+
+ // We can only tolerate being cancelled during concurrent marking or during preparation for mixed
+ // evacuation. This flag here (passed by reference) is used to control precisely where the regulator
+ // is allowed to cancel a GC.
+ ShenandoahOldGC gc(generation, _allow_old_preemption);
+ if (gc.collect(cause)) {
+ heap->notify_gc_progress();
+ generation->record_success_concurrent(false);
+ }
+
+ if (heap->cancelled_gc()) {
+ // It's possible the gc cycle was cancelled after the last time
+ // the collection checked for cancellation. In which case, the
+ // old gc cycle is still completed, and we have to deal with this
+ // cancellation. We set the degeneration point to be outside
+ // the cycle because if this is an allocation failure, that is
+ // what must be done (there is no degenerated old cycle). If the
+ // cancellation was due to a heuristic wanting to start a young
+ // cycle, then we are not actually going to a degenerated cycle,
+ // so the degenerated point doesn't matter here.
+ check_cancellation_or_degen(ShenandoahGC::_degenerated_outside_cycle);
+ if (_requested_gc_cause == GCCause::_shenandoah_concurrent_gc) {
+ heap->shenandoah_policy()->record_interrupted_old();
+ }
+ return false;
+ }
+ return true;
+}
+
+void ShenandoahGenerationalControlThread::service_concurrent_cycle(ShenandoahGeneration* generation, GCCause::Cause cause, bool do_old_gc_bootstrap) {
+ // Normal cycle goes via all concurrent phases. If allocation failure (af) happens during
+ // any of the concurrent phases, it first degrades to Degenerated GC and completes GC there.
+ // If second allocation failure happens during Degenerated GC cycle (for example, when GC
+ // tries to evac something and no memory is available), cycle degrades to Full GC.
+ //
+ // There are also a shortcut through the normal cycle: immediate garbage shortcut, when
+ // heuristics says there are no regions to compact, and all the collection comes from immediately
+ // reclaimable regions.
+ //
+ // ................................................................................................
+ //
+ // (immediate garbage shortcut) Concurrent GC
+ // /-------------------------------------------\
+ // | |
+ // | |
+ // | |
+ // | v
+ // [START] ----> Conc Mark ----o----> Conc Evac --o--> Conc Update-Refs ---o----> [END]
+ // | | | ^
+ // | (af) | (af) | (af) |
+ // ..................|....................|.................|..............|.......................
+ // | | | |
+ // | | | | Degenerated GC
+ // v v v |
+ // STW Mark ----------> STW Evac ----> STW Update-Refs ----->o
+ // | | | ^
+ // | (af) | (af) | (af) |
+ // ..................|....................|.................|..............|.......................
+ // | | | |
+ // | v | | Full GC
+ // \------------------->o<----------------/ |
+ // | |
+ // v |
+ // Full GC --------------------------/
+ //
+ if (check_cancellation_or_degen(ShenandoahGC::_degenerated_outside_cycle)) return;
+
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+ ShenandoahGCSession session(cause, generation);
+ TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
+
+ service_concurrent_cycle(heap, generation, cause, do_old_gc_bootstrap);
+}
+
+void ShenandoahGenerationalControlThread::service_concurrent_cycle(ShenandoahHeap* heap,
+ ShenandoahGeneration* generation,
+ GCCause::Cause& cause,
+ bool do_old_gc_bootstrap) {
+ assert(!generation->is_old(), "Old GC takes a different control path");
+
+ ShenandoahConcurrentGC gc(generation, do_old_gc_bootstrap);
+ if (gc.collect(cause)) {
+ // Cycle is complete
+ heap->notify_gc_progress();
+ generation->record_success_concurrent(gc.abbreviated());
+ } else {
+ assert(heap->cancelled_gc(), "Must have been cancelled");
+ check_cancellation_or_degen(gc.degen_point());
+
+ // Concurrent young-gen collection degenerates to young
+ // collection. Same for global collections.
+ _degen_generation = generation;
+ }
+ const char* msg;
+ ShenandoahMmuTracker* mmu_tracker = heap->mmu_tracker();
+ if (generation->is_young()) {
+ if (heap->cancelled_gc()) {
+ msg = (do_old_gc_bootstrap) ? "At end of Interrupted Concurrent Bootstrap GC" :
+ "At end of Interrupted Concurrent Young GC";
+ } else {
+ // We only record GC results if GC was successful
+ msg = (do_old_gc_bootstrap) ? "At end of Concurrent Bootstrap GC" :
+ "At end of Concurrent Young GC";
+ if (heap->collection_set()->has_old_regions()) {
+ mmu_tracker->record_mixed(get_gc_id());
+ } else if (do_old_gc_bootstrap) {
+ mmu_tracker->record_bootstrap(get_gc_id());
+ } else {
+ mmu_tracker->record_young(get_gc_id());
+ }
+ }
+ } else {
+ assert(generation->is_global(), "If not young, must be GLOBAL");
+ assert(!do_old_gc_bootstrap, "Do not bootstrap with GLOBAL GC");
+ if (heap->cancelled_gc()) {
+ msg = "At end of Interrupted Concurrent GLOBAL GC";
+ } else {
+ // We only record GC results if GC was successful
+ msg = "At end of Concurrent Global GC";
+ mmu_tracker->record_global(get_gc_id());
+ }
+ }
+ heap->log_heap_status(msg);
+}
+
+bool ShenandoahGenerationalControlThread::check_cancellation_or_degen(ShenandoahGC::ShenandoahDegenPoint point) {
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+ if (!heap->cancelled_gc()) {
+ return false;
+ }
+
+ if (in_graceful_shutdown()) {
+ return true;
+ }
+
+ assert(_degen_point == ShenandoahGC::_degenerated_outside_cycle,
+ "Should not be set yet: %s", ShenandoahGC::degen_point_to_string(_degen_point));
+
+ if (is_alloc_failure_gc()) {
+ _degen_point = point;
+ _preemption_requested.unset();
+ return true;
+ }
+
+ if (_preemption_requested.is_set()) {
+ assert(_requested_generation == YOUNG, "Only young GCs may preempt old.");
+ _preemption_requested.unset();
+
+ // Old generation marking is only cancellable during concurrent marking.
+ // Once final mark is complete, the code does not check again for cancellation.
+ // If old generation was cancelled for an allocation failure, we wouldn't
+ // make it to this case. The calling code is responsible for forcing a
+ // cancellation due to allocation failure into a degenerated cycle.
+ _degen_point = point;
+ heap->clear_cancelled_gc(false /* clear oom handler */);
+ return true;
+ }
+
+ fatal("Cancel GC either for alloc failure GC, or gracefully exiting, or to pause old generation marking");
+ return false;
+}
+
+void ShenandoahGenerationalControlThread::stop_service() {
+ // Nothing to do here.
+}
+
+void ShenandoahGenerationalControlThread::service_stw_full_cycle(GCCause::Cause cause) {
+ ShenandoahHeap* const heap = ShenandoahHeap::heap();
+
+ GCIdMark gc_id_mark;
+ ShenandoahGCSession session(cause, heap->global_generation());
+
+ ShenandoahFullGC gc;
+ gc.collect(cause);
+}
+
+void ShenandoahGenerationalControlThread::service_stw_degenerated_cycle(GCCause::Cause cause,
+ ShenandoahGC::ShenandoahDegenPoint point) {
+ assert(point != ShenandoahGC::_degenerated_unset, "Degenerated point should be set");
+ ShenandoahHeap* const heap = ShenandoahHeap::heap();
+
+ GCIdMark gc_id_mark;
+ ShenandoahGCSession session(cause, _degen_generation);
+
+ ShenandoahDegenGC gc(point, _degen_generation);
+ gc.collect(cause);
+
+ assert(heap->young_generation()->task_queues()->is_empty(), "Unexpected young generation marking tasks");
+ if (_degen_generation->is_global()) {
+ assert(heap->old_generation()->task_queues()->is_empty(), "Unexpected old generation marking tasks");
+ assert(heap->global_generation()->task_queues()->is_empty(), "Unexpected global generation marking tasks");
+ } else {
+ assert(_degen_generation->is_young(), "Expected degenerated young cycle, if not global.");
+ ShenandoahOldGeneration* old = heap->old_generation();
+ if (old->is_bootstrapping()) {
+ old->transition_to(ShenandoahOldGeneration::MARKING);
+ }
+ }
+}
+
+void ShenandoahGenerationalControlThread::request_gc(GCCause::Cause cause) {
+ if (ShenandoahCollectorPolicy::should_handle_requested_gc(cause)) {
+ handle_requested_gc(cause);
+ }
+}
+
+bool ShenandoahGenerationalControlThread::request_concurrent_gc(ShenandoahGenerationType generation) {
+ if (_preemption_requested.is_set() || _requested_gc_cause != GCCause::_no_gc || ShenandoahHeap::heap()->cancelled_gc()) {
+ // Ignore subsequent requests from the heuristics
+ log_debug(gc, thread)("Reject request for concurrent gc: preemption_requested: %s, gc_requested: %s, gc_cancelled: %s",
+ BOOL_TO_STR(_preemption_requested.is_set()),
+ GCCause::to_string(_requested_gc_cause),
+ BOOL_TO_STR(ShenandoahHeap::heap()->cancelled_gc()));
+ return false;
+ }
+
+ if (gc_mode() == none) {
+ GCCause::Cause existing = Atomic::cmpxchg(&_requested_gc_cause, GCCause::_no_gc, GCCause::_shenandoah_concurrent_gc);
+ if (existing != GCCause::_no_gc) {
+ log_debug(gc, thread)("Reject request for concurrent gc because another gc is pending: %s", GCCause::to_string(existing));
+ return false;
+ }
+
+ _requested_generation = generation;
+ notify_control_thread();
+
+ MonitorLocker ml(&_regulator_lock, Mutex::_no_safepoint_check_flag);
+ while (gc_mode() == none) {
+ ml.wait();
+ }
+ return true;
+ }
+
+ if (preempt_old_marking(generation)) {
+ assert(gc_mode() == servicing_old, "Expected to be servicing old, but was: %s.", gc_mode_name(gc_mode()));
+ GCCause::Cause existing = Atomic::cmpxchg(&_requested_gc_cause, GCCause::_no_gc, GCCause::_shenandoah_concurrent_gc);
+ if (existing != GCCause::_no_gc) {
+ log_debug(gc, thread)("Reject request to interrupt old gc because another gc is pending: %s", GCCause::to_string(existing));
+ return false;
+ }
+
+ log_info(gc)("Preempting old generation mark to allow %s GC", shenandoah_generation_name(generation));
+ _requested_generation = generation;
+ _preemption_requested.set();
+ ShenandoahHeap::heap()->cancel_gc(GCCause::_shenandoah_concurrent_gc);
+ notify_control_thread();
+
+ MonitorLocker ml(&_regulator_lock, Mutex::_no_safepoint_check_flag);
+ while (gc_mode() == servicing_old) {
+ ml.wait();
+ }
+ return true;
+ }
+
+ log_debug(gc, thread)("Reject request for concurrent gc: mode: %s, allow_old_preemption: %s",
+ gc_mode_name(gc_mode()),
+ BOOL_TO_STR(_allow_old_preemption.is_set()));
+ return false;
+}
+
+void ShenandoahGenerationalControlThread::notify_control_thread() {
+ MonitorLocker locker(&_control_lock, Mutex::_no_safepoint_check_flag);
+ _control_lock.notify();
+}
+
+bool ShenandoahGenerationalControlThread::preempt_old_marking(ShenandoahGenerationType generation) {
+ return (generation == YOUNG) && _allow_old_preemption.try_unset();
+}
+
+void ShenandoahGenerationalControlThread::handle_requested_gc(GCCause::Cause cause) {
+ // For normal requested GCs (System.gc) we want to block the caller. However,
+ // for whitebox requested GC, we want to initiate the GC and return immediately.
+ // The whitebox caller thread will arrange for itself to wait until the GC notifies
+ // it that has reached the requested breakpoint (phase in the GC).
+ if (cause == GCCause::_wb_breakpoint) {
+ Atomic::xchg(&_requested_gc_cause, cause);
+ notify_control_thread();
+ return;
+ }
+
+ // Make sure we have at least one complete GC cycle before unblocking
+ // from the explicit GC request.
+ //
+ // This is especially important for weak references cleanup and/or native
+ // resources (e.g. DirectByteBuffers) machinery: when explicit GC request
+ // comes very late in the already running cycle, it would miss lots of new
+ // opportunities for cleanup that were made available before the caller
+ // requested the GC.
+
+ MonitorLocker ml(&_gc_waiters_lock);
+ size_t current_gc_id = get_gc_id();
+ size_t required_gc_id = current_gc_id + 1;
+ while (current_gc_id < required_gc_id) {
+ // This races with the regulator thread to start a concurrent gc and the
+ // control thread to clear it at the start of a cycle. Threads here are
+ // allowed to escalate a heuristic's request for concurrent gc.
+ GCCause::Cause existing = Atomic::xchg(&_requested_gc_cause, cause);
+ if (existing != GCCause::_no_gc) {
+ log_debug(gc, thread)("GC request supersedes existing request: %s", GCCause::to_string(existing));
+ }
+
+ notify_control_thread();
+ ml.wait();
+ current_gc_id = get_gc_id();
+ }
+}
+
+void ShenandoahGenerationalControlThread::notify_gc_waiters() {
+ MonitorLocker ml(&_gc_waiters_lock);
+ ml.notify_all();
+}
+
+const char* ShenandoahGenerationalControlThread::gc_mode_name(ShenandoahGenerationalControlThread::GCMode mode) {
+ switch (mode) {
+ case none: return "idle";
+ case concurrent_normal: return "normal";
+ case stw_degenerated: return "degenerated";
+ case stw_full: return "full";
+ case servicing_old: return "old";
+ case bootstrapping_old: return "bootstrap";
+ default: return "unknown";
+ }
+}
+
+void ShenandoahGenerationalControlThread::set_gc_mode(ShenandoahGenerationalControlThread::GCMode new_mode) {
+ if (_mode != new_mode) {
+ log_debug(gc)("Transition from: %s to: %s", gc_mode_name(_mode), gc_mode_name(new_mode));
+ MonitorLocker ml(&_regulator_lock, Mutex::_no_safepoint_check_flag);
+ _mode = new_mode;
+ ml.notify_all();
+ }
+}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.hpp
new file mode 100644
index 00000000000..295882fe6d9
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.hpp
@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONALCONTROLTHREAD_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONALCONTROLTHREAD_HPP
+
+#include "gc/shared/gcCause.hpp"
+#include "gc/shenandoah/shenandoahController.hpp"
+#include "gc/shenandoah/shenandoahGenerationType.hpp"
+#include "gc/shenandoah/shenandoahGC.hpp"
+#include "gc/shenandoah/shenandoahPadding.hpp"
+#include "gc/shenandoah/shenandoahSharedVariables.hpp"
+
+class ShenandoahOldGeneration;
+class ShenandoahGeneration;
+class ShenandoahGenerationalHeap;
+class ShenandoahHeap;
+
+class ShenandoahGenerationalControlThread: public ShenandoahController {
+ friend class VMStructs;
+
+public:
+ typedef enum {
+ none,
+ concurrent_normal,
+ stw_degenerated,
+ stw_full,
+ bootstrapping_old,
+ servicing_old
+ } GCMode;
+
+private:
+ Monitor _control_lock;
+ Monitor _regulator_lock;
+
+ ShenandoahSharedFlag _allow_old_preemption;
+ ShenandoahSharedFlag _preemption_requested;
+
+ GCCause::Cause _requested_gc_cause;
+ volatile ShenandoahGenerationType _requested_generation;
+ ShenandoahGC::ShenandoahDegenPoint _degen_point;
+ ShenandoahGeneration* _degen_generation;
+
+ shenandoah_padding(0);
+ volatile GCMode _mode;
+ shenandoah_padding(1);
+
+public:
+ ShenandoahGenerationalControlThread();
+
+ void run_service() override;
+ void stop_service() override;
+
+ void request_gc(GCCause::Cause cause) override;
+
+ // Return true if the request to start a concurrent GC for the given generation succeeded.
+ bool request_concurrent_gc(ShenandoahGenerationType generation);
+
+ GCMode gc_mode() {
+ return _mode;
+ }
+private:
+
+ // Returns true if the cycle has been cancelled or degenerated.
+ bool check_cancellation_or_degen(ShenandoahGC::ShenandoahDegenPoint point);
+
+ // Returns true if the old generation marking completed (i.e., final mark executed for old generation).
+ bool resume_concurrent_old_cycle(ShenandoahOldGeneration* generation, GCCause::Cause cause);
+ void service_concurrent_cycle(ShenandoahGeneration* generation, GCCause::Cause cause, bool reset_old_bitmap_specially);
+ void service_stw_full_cycle(GCCause::Cause cause);
+ void service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahGC::ShenandoahDegenPoint point);
+
+ void notify_gc_waiters();
+
+ // Handle GC request.
+ // Blocks until GC is over.
+ void handle_requested_gc(GCCause::Cause cause);
+
+ bool is_explicit_gc(GCCause::Cause cause) const;
+ bool is_implicit_gc(GCCause::Cause cause) const;
+
+ // Returns true if the old generation marking was interrupted to allow a young cycle.
+ bool preempt_old_marking(ShenandoahGenerationType generation);
+
+ void process_phase_timings(const ShenandoahGenerationalHeap* heap);
+
+ void service_concurrent_normal_cycle(ShenandoahGenerationalHeap* heap,
+ ShenandoahGenerationType generation,
+ GCCause::Cause cause);
+
+ void service_concurrent_old_cycle(ShenandoahGenerationalHeap* heap,
+ GCCause::Cause &cause);
+
+ void set_gc_mode(GCMode new_mode);
+
+ static const char* gc_mode_name(GCMode mode);
+
+ void notify_control_thread();
+
+ void service_concurrent_cycle(ShenandoahHeap* heap,
+ ShenandoahGeneration* generation,
+ GCCause::Cause &cause,
+ bool do_old_gc_bootstrap);
+
+};
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONALCONTROLTHREAD_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp
new file mode 100644
index 00000000000..81bb5c56a86
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp
@@ -0,0 +1,326 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/shenandoah/shenandoahAsserts.hpp"
+#include "gc/shenandoah/shenandoahFreeSet.hpp"
+#include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
+#include "gc/shenandoah/shenandoahPacer.hpp"
+#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
+#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
+#include "gc/shenandoah/shenandoahUtils.hpp"
+
+class ShenandoahConcurrentEvacuator : public ObjectClosure {
+private:
+ ShenandoahGenerationalHeap* const _heap;
+ Thread* const _thread;
+public:
+ explicit ShenandoahConcurrentEvacuator(ShenandoahGenerationalHeap* heap) :
+ _heap(heap), _thread(Thread::current()) {}
+
+ void do_object(oop p) override {
+ shenandoah_assert_marked(nullptr, p);
+ if (!p->is_forwarded()) {
+ _heap->evacuate_object(p, _thread);
+ }
+ }
+};
+
+ShenandoahGenerationalEvacuationTask::ShenandoahGenerationalEvacuationTask(ShenandoahGenerationalHeap* heap,
+ ShenandoahRegionIterator* iterator,
+ bool concurrent, bool only_promote_regions) :
+ WorkerTask("Shenandoah Evacuation"),
+ _heap(heap),
+ _regions(iterator),
+ _concurrent(concurrent),
+ _only_promote_regions(only_promote_regions),
+ _tenuring_threshold(0)
+{
+ shenandoah_assert_generational();
+ _tenuring_threshold = _heap->age_census()->tenuring_threshold();
+}
+
+void ShenandoahGenerationalEvacuationTask::work(uint worker_id) {
+ if (_concurrent) {
+ ShenandoahConcurrentWorkerSession worker_session(worker_id);
+ ShenandoahSuspendibleThreadSetJoiner stsj;
+ do_work();
+ } else {
+ ShenandoahParallelWorkerSession worker_session(worker_id);
+ do_work();
+ }
+}
+
+void ShenandoahGenerationalEvacuationTask::do_work() {
+ if (_only_promote_regions) {
+ // No allocations will be made, do not enter oom-during-evac protocol.
+ assert(ShenandoahHeap::heap()->collection_set()->is_empty(), "Should not have a collection set here");
+ promote_regions();
+ } else {
+ assert(!ShenandoahHeap::heap()->collection_set()->is_empty(), "Should have a collection set here");
+ ShenandoahEvacOOMScope oom_evac_scope;
+ evacuate_and_promote_regions();
+ }
+}
+
+void log_region(const ShenandoahHeapRegion* r, LogStream* ls) {
+ ls->print_cr("GenerationalEvacuationTask, looking at %s region " SIZE_FORMAT ", (age: %d) [%s, %s, %s]",
+ r->is_old()? "old": r->is_young()? "young": "free", r->index(), r->age(),
+ r->is_active()? "active": "inactive",
+ r->is_humongous()? (r->is_humongous_start()? "humongous_start": "humongous_continuation"): "regular",
+ r->is_cset()? "cset": "not-cset");
+}
+
+void ShenandoahGenerationalEvacuationTask::promote_regions() {
+ ShenandoahHeapRegion* r;
+ LogTarget(Debug, gc) lt;
+
+ while ((r = _regions->next()) != nullptr) {
+ if (lt.is_enabled()) {
+ LogStream ls(lt);
+ log_region(r, &ls);
+ }
+
+ maybe_promote_region(r);
+
+ if (_heap->check_cancelled_gc_and_yield(_concurrent)) {
+ break;
+ }
+ }
+}
+
+void ShenandoahGenerationalEvacuationTask::evacuate_and_promote_regions() {
+ LogTarget(Debug, gc) lt;
+ ShenandoahConcurrentEvacuator cl(_heap);
+ ShenandoahHeapRegion* r;
+
+ while ((r = _regions->next()) != nullptr) {
+ if (lt.is_enabled()) {
+ LogStream ls(lt);
+ log_region(r, &ls);
+ }
+
+ if (r->is_cset()) {
+ assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
+ _heap->marked_object_iterate(r, &cl);
+ if (ShenandoahPacing) {
+ _heap->pacer()->report_evac(r->used() >> LogHeapWordSize);
+ }
+ } else {
+ maybe_promote_region(r);
+ }
+
+ if (_heap->check_cancelled_gc_and_yield(_concurrent)) {
+ break;
+ }
+ }
+}
+
+
+void ShenandoahGenerationalEvacuationTask::maybe_promote_region(ShenandoahHeapRegion* r) {
+ if (r->is_young() && r->is_active() && (r->age() >= _tenuring_threshold)) {
+ if (r->is_humongous_start()) {
+ // We promote humongous_start regions along with their affiliated continuations during evacuation rather than
+ // doing this work during a safepoint. We cannot put humongous regions into the collection set because that
+ // triggers the load-reference barrier (LRB) to copy on reference fetch.
+ //
+ // Aged humongous continuation regions are handled with their start region. If an aged regular region has
+ // more garbage than ShenandoahOldGarbageThreshold, we'll promote by evacuation. If there is room for evacuation
+ // in this cycle, the region will be in the collection set. If there is not room, the region will be promoted
+ // by evacuation in some future GC cycle.
+ promote_humongous(r);
+ } else if (r->is_regular() && (r->get_top_before_promote() != nullptr)) {
+ // Likewise, we cannot put promote-in-place regions into the collection set because that would also trigger
+ // the LRB to copy on reference fetch.
+ //
+ // If an aged regular region has received allocations during the current cycle, we do not promote because the
+ // newly allocated objects do not have appropriate age; this region's age will be reset to zero at end of cycle.
+ promote_in_place(r);
+ }
+ }
+}
+
+// When we promote a region in place, we can continue to use the established marking context to guide subsequent remembered
+// set scans of this region's content. The region will be coalesced and filled prior to the next old-gen marking effort.
+// We identify the entirety of the region as DIRTY to force the next remembered set scan to identify the "interesting pointers"
+// contained herein.
+void ShenandoahGenerationalEvacuationTask::promote_in_place(ShenandoahHeapRegion* region) {
+ ShenandoahMarkingContext* const marking_context = _heap->complete_marking_context();
+ HeapWord* const tams = marking_context->top_at_mark_start(region);
+
+ {
+ const size_t old_garbage_threshold = (ShenandoahHeapRegion::region_size_bytes() * ShenandoahOldGarbageThreshold) / 100;
+ shenandoah_assert_generations_reconciled();
+ assert(!_heap->is_concurrent_old_mark_in_progress(), "Cannot promote in place during old marking");
+ assert(region->garbage_before_padded_for_promote() < old_garbage_threshold, "Region " SIZE_FORMAT " has too much garbage for promotion", region->index());
+ assert(region->is_young(), "Only young regions can be promoted");
+ assert(region->is_regular(), "Use different service to promote humongous regions");
+ assert(region->age() >= _tenuring_threshold, "Only promote regions that are sufficiently aged");
+ assert(region->get_top_before_promote() == tams, "Region " SIZE_FORMAT " has been used for allocations before promotion", region->index());
+ }
+
+ ShenandoahOldGeneration* const old_gen = _heap->old_generation();
+ ShenandoahYoungGeneration* const young_gen = _heap->young_generation();
+
+ // Rebuild the remembered set information and mark the entire range as DIRTY. We do NOT scan the content of this
+ // range to determine which cards need to be DIRTY. That would force us to scan the region twice, once now, and
+ // once during the subsequent remembered set scan. Instead, we blindly (conservatively) mark everything as DIRTY
+ // now and then sort out the CLEAN pages during the next remembered set scan.
+ //
+ // Rebuilding the remembered set consists of clearing all object registrations (reset_object_range()) here,
+ // then registering every live object and every coalesced range of free objects in the loop that follows.
+ ShenandoahScanRemembered* const scanner = old_gen->card_scan();
+ scanner->reset_object_range(region->bottom(), region->end());
+ scanner->mark_range_as_dirty(region->bottom(), region->get_top_before_promote() - region->bottom());
+
+ HeapWord* obj_addr = region->bottom();
+ while (obj_addr < tams) {
+ oop obj = cast_to_oop(obj_addr);
+ if (marking_context->is_marked(obj)) {
+ assert(obj->klass() != nullptr, "klass should not be NULL");
+ // This thread is responsible for registering all objects in this region. No need for lock.
+ scanner->register_object_without_lock(obj_addr);
+ obj_addr += obj->size();
+ } else {
+ HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, tams);
+ assert(next_marked_obj <= tams, "next marked object cannot exceed tams");
+ size_t fill_size = next_marked_obj - obj_addr;
+ assert(fill_size >= ShenandoahHeap::min_fill_size(), "previously allocated objects known to be larger than min_size");
+ ShenandoahHeap::fill_with_object(obj_addr, fill_size);
+ scanner->register_object_without_lock(obj_addr);
+ obj_addr = next_marked_obj;
+ }
+ }
+ // We do not need to scan above TAMS because restored top equals tams
+ assert(obj_addr == tams, "Expect loop to terminate when obj_addr equals tams");
+
+
+ {
+ ShenandoahHeapLocker locker(_heap->lock());
+
+ HeapWord* update_watermark = region->get_update_watermark();
+
+ // Now that this region is affiliated with old, we can allow it to receive allocations, though it may not be in the
+ // is_collector_free range.
+ region->restore_top_before_promote();
+
+ size_t region_used = region->used();
+
+ // The update_watermark was likely established while we had the artificially high value of top. Make it sane now.
+ assert(update_watermark >= region->top(), "original top cannot exceed preserved update_watermark");
+ region->set_update_watermark(region->top());
+
+ // Unconditionally transfer one region from young to old. This represents the newly promoted region.
+ // This expands old and shrinks new by the size of one region. Strictly, we do not "need" to expand old
+ // if there are already enough unaffiliated regions in old to account for this newly promoted region.
+ // However, if we do not transfer the capacities, we end up reducing the amount of memory that would have
+ // otherwise been available to hold old evacuations, because old available is max_capacity - used and now
+ // we would be trading a fully empty region for a partially used region.
+ young_gen->decrease_used(region_used);
+ young_gen->decrement_affiliated_region_count();
+
+ // transfer_to_old() increases capacity of old and decreases capacity of young
+ _heap->generation_sizer()->force_transfer_to_old(1);
+ region->set_affiliation(OLD_GENERATION);
+
+ old_gen->increment_affiliated_region_count();
+ old_gen->increase_used(region_used);
+
+ // add_old_collector_free_region() increases promoted_reserve() if available space exceeds plab_min_size()
+ _heap->free_set()->add_promoted_in_place_region_to_old_collector(region);
+ }
+}
+
+void ShenandoahGenerationalEvacuationTask::promote_humongous(ShenandoahHeapRegion* region) {
+ ShenandoahMarkingContext* marking_context = _heap->marking_context();
+ oop obj = cast_to_oop(region->bottom());
+ assert(_heap->gc_generation()->is_mark_complete(), "sanity");
+ shenandoah_assert_generations_reconciled();
+ assert(region->is_young(), "Only young regions can be promoted");
+ assert(region->is_humongous_start(), "Should not promote humongous continuation in isolation");
+ assert(region->age() >= _tenuring_threshold, "Only promote regions that are sufficiently aged");
+ assert(marking_context->is_marked(obj), "promoted humongous object should be alive");
+
+ const size_t used_bytes = obj->size() * HeapWordSize;
+ const size_t spanned_regions = ShenandoahHeapRegion::required_regions(used_bytes);
+ const size_t humongous_waste = spanned_regions * ShenandoahHeapRegion::region_size_bytes() - obj->size() * HeapWordSize;
+ const size_t index_limit = region->index() + spanned_regions;
+
+ ShenandoahOldGeneration* const old_gen = _heap->old_generation();
+ ShenandoahGeneration* const young_gen = _heap->young_generation();
+ {
+ // We need to grab the heap lock in order to avoid a race when changing the affiliations of spanned_regions from
+ // young to old.
+ ShenandoahHeapLocker locker(_heap->lock());
+
+ // We promote humongous objects unconditionally, without checking for availability. We adjust
+ // usage totals, including humongous waste, after evacuation is done.
+ log_debug(gc)("promoting humongous region " SIZE_FORMAT ", spanning " SIZE_FORMAT, region->index(), spanned_regions);
+
+ young_gen->decrease_used(used_bytes);
+ young_gen->decrease_humongous_waste(humongous_waste);
+ young_gen->decrease_affiliated_region_count(spanned_regions);
+
+ // transfer_to_old() increases capacity of old and decreases capacity of young
+ _heap->generation_sizer()->force_transfer_to_old(spanned_regions);
+
+ // For this region and each humongous continuation region spanned by this humongous object, change
+ // affiliation to OLD_GENERATION and adjust the generation-use tallies. The remnant of memory
+ // in the last humongous region that is not spanned by obj is currently not used.
+ for (size_t i = region->index(); i < index_limit; i++) {
+ ShenandoahHeapRegion* r = _heap->get_region(i);
+ log_debug(gc)("promoting humongous region " SIZE_FORMAT ", from " PTR_FORMAT " to " PTR_FORMAT,
+ r->index(), p2i(r->bottom()), p2i(r->top()));
+ // We mark the entire humongous object's range as dirty after loop terminates, so no need to dirty the range here
+ r->set_affiliation(OLD_GENERATION);
+ }
+
+ old_gen->increase_affiliated_region_count(spanned_regions);
+ old_gen->increase_used(used_bytes);
+ old_gen->increase_humongous_waste(humongous_waste);
+ }
+
+ // Since this region may have served previously as OLD, it may hold obsolete object range info.
+ HeapWord* const humongous_bottom = region->bottom();
+ ShenandoahScanRemembered* const scanner = old_gen->card_scan();
+ scanner->reset_object_range(humongous_bottom, humongous_bottom + spanned_regions * ShenandoahHeapRegion::region_size_words());
+ // Since the humongous region holds only one object, no lock is necessary for this register_object() invocation.
+ scanner->register_object_without_lock(humongous_bottom);
+
+ if (obj->is_typeArray()) {
+ // Primitive arrays don't need to be scanned.
+ log_debug(gc)("Clean cards for promoted humongous object (Region " SIZE_FORMAT ") from " PTR_FORMAT " to " PTR_FORMAT,
+ region->index(), p2i(humongous_bottom), p2i(humongous_bottom + obj->size()));
+ scanner->mark_range_as_clean(humongous_bottom, obj->size());
+ } else {
+ log_debug(gc)("Dirty cards for promoted humongous object (Region " SIZE_FORMAT ") from " PTR_FORMAT " to " PTR_FORMAT,
+ region->index(), p2i(humongous_bottom), p2i(humongous_bottom + obj->size()));
+ scanner->mark_range_as_dirty(humongous_bottom, obj->size());
+ }
+}
+
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp
new file mode 100644
index 00000000000..abe2fc0110c
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp
@@ -0,0 +1,58 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONALEVACUATIONTASK_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONALEVACUATIONTASK_HPP
+
+#include "gc/shared/workerThread.hpp"
+
+class ShenandoahGenerationalHeap;
+class ShenandoahHeapRegion;
+class ShenandoahRegionIterator;
+
+// Unlike ShenandoahEvacuationTask, this iterates over all regions rather than just the collection set.
+// This is needed in order to promote humongous start regions if age() >= tenure threshold.
+class ShenandoahGenerationalEvacuationTask : public WorkerTask {
+private:
+ ShenandoahGenerationalHeap* const _heap;
+ ShenandoahRegionIterator* _regions;
+ bool _concurrent;
+ bool _only_promote_regions;
+ uint _tenuring_threshold;
+
+public:
+ ShenandoahGenerationalEvacuationTask(ShenandoahGenerationalHeap* sh,
+ ShenandoahRegionIterator* iterator,
+ bool concurrent, bool only_promote_regions);
+ void work(uint worker_id) override;
+private:
+ void do_work();
+ void promote_regions();
+ void evacuate_and_promote_regions();
+ void maybe_promote_region(ShenandoahHeapRegion* region);
+ void promote_in_place(ShenandoahHeapRegion* region);
+ void promote_humongous(ShenandoahHeapRegion* region);
+};
+
+#endif //SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONALEVACUATIONTASK_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.cpp
new file mode 100644
index 00000000000..fe38c996bd8
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.cpp
@@ -0,0 +1,390 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+
+#include "precompiled.hpp"
+
+#include "gc/shared/fullGCForwarding.inline.hpp"
+#include "gc/shared/preservedMarks.inline.hpp"
+#include "gc/shenandoah/shenandoahGenerationalFullGC.hpp"
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
+#include "gc/shenandoah/shenandoahGeneration.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
+#include "gc/shenandoah/shenandoahUtils.hpp"
+
+#ifdef ASSERT
+void assert_regions_used_not_more_than_capacity(ShenandoahGeneration* generation) {
+ assert(generation->used_regions_size() <= generation->max_capacity(),
+ "%s generation affiliated regions must be less than capacity", generation->name());
+}
+
+void assert_usage_not_more_than_regions_used(ShenandoahGeneration* generation) {
+ assert(generation->used_including_humongous_waste() <= generation->used_regions_size(),
+ "%s consumed can be no larger than span of affiliated regions", generation->name());
+}
+#else
+void assert_regions_used_not_more_than_capacity(ShenandoahGeneration* generation) {}
+void assert_usage_not_more_than_regions_used(ShenandoahGeneration* generation) {}
+#endif
+
+
+void ShenandoahGenerationalFullGC::prepare() {
+ auto heap = ShenandoahGenerationalHeap::heap();
+ // Since we may arrive here from degenerated GC failure of either young or old, establish generation as GLOBAL.
+ heap->set_gc_generation(heap->global_generation());
+ heap->set_active_generation();
+
+ // No need for old_gen->increase_used() as this was done when plabs were allocated.
+ heap->reset_generation_reserves();
+
+ // Full GC supersedes any marking or coalescing in old generation.
+ heap->old_generation()->cancel_gc();
+}
+
+void ShenandoahGenerationalFullGC::handle_completion(ShenandoahHeap* heap) {
+ // Full GC should reset time since last gc for young and old heuristics
+ ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::cast(heap);
+ ShenandoahYoungGeneration* young = gen_heap->young_generation();
+ ShenandoahOldGeneration* old = gen_heap->old_generation();
+ young->heuristics()->record_cycle_end();
+ old->heuristics()->record_cycle_end();
+
+ gen_heap->mmu_tracker()->record_full(GCId::current());
+ gen_heap->log_heap_status("At end of Full GC");
+
+ assert(old->is_idle(), "After full GC, old generation should be idle.");
+
+ // Since we allow temporary violation of these constraints during Full GC, we want to enforce that the assertions are
+ // made valid by the time Full GC completes.
+ assert_regions_used_not_more_than_capacity(old);
+ assert_regions_used_not_more_than_capacity(young);
+ assert_usage_not_more_than_regions_used(old);
+ assert_usage_not_more_than_regions_used(young);
+
+ // Establish baseline for next old-has-grown trigger.
+ old->set_live_bytes_after_last_mark(old->used_including_humongous_waste());
+}
+
+void ShenandoahGenerationalFullGC::rebuild_remembered_set(ShenandoahHeap* heap) {
+ ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_reconstruct_remembered_set);
+ ShenandoahRegionIterator regions;
+ ShenandoahReconstructRememberedSetTask task(®ions);
+ heap->workers()->run_task(&task);
+
+ // Rebuilding the remembered set recomputes all the card offsets for objects.
+ // The adjust pointers phase coalesces and fills all necessary regions. In case
+ // we came to the full GC from an incomplete global cycle, we need to indicate
+ // that the old regions are parsable.
+ heap->old_generation()->set_parsable(true);
+}
+
+void ShenandoahGenerationalFullGC::balance_generations_after_gc(ShenandoahHeap* heap) {
+ ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::cast(heap);
+ ShenandoahOldGeneration* const old_gen = gen_heap->old_generation();
+
+ size_t old_usage = old_gen->used_regions_size();
+ size_t old_capacity = old_gen->max_capacity();
+
+ assert(old_usage % ShenandoahHeapRegion::region_size_bytes() == 0, "Old usage must align with region size");
+ assert(old_capacity % ShenandoahHeapRegion::region_size_bytes() == 0, "Old capacity must align with region size");
+
+ if (old_capacity > old_usage) {
+ size_t excess_old_regions = (old_capacity - old_usage) / ShenandoahHeapRegion::region_size_bytes();
+ gen_heap->generation_sizer()->transfer_to_young(excess_old_regions);
+ } else if (old_capacity < old_usage) {
+ size_t old_regions_deficit = (old_usage - old_capacity) / ShenandoahHeapRegion::region_size_bytes();
+ gen_heap->generation_sizer()->force_transfer_to_old(old_regions_deficit);
+ }
+
+ log_info(gc, ergo)("FullGC done: young usage: " PROPERFMT ", old usage: " PROPERFMT,
+ PROPERFMTARGS(gen_heap->young_generation()->used()),
+ PROPERFMTARGS(old_gen->used()));
+}
+
+void ShenandoahGenerationalFullGC::balance_generations_after_rebuilding_free_set() {
+ auto result = ShenandoahGenerationalHeap::heap()->balance_generations();
+ LogTarget(Info, gc, ergo) lt;
+ if (lt.is_enabled()) {
+ LogStream ls(lt);
+ result.print_on("Full GC", &ls);
+ }
+}
+
+void ShenandoahGenerationalFullGC::log_live_in_old(ShenandoahHeap* heap) {
+ LogTarget(Debug, gc) lt;
+ if (lt.is_enabled()) {
+ size_t live_bytes_in_old = 0;
+ for (size_t i = 0; i < heap->num_regions(); i++) {
+ ShenandoahHeapRegion* r = heap->get_region(i);
+ if (r->is_old()) {
+ live_bytes_in_old += r->get_live_data_bytes();
+ }
+ }
+ log_debug(gc)("Live bytes in old after STW mark: " PROPERFMT, PROPERFMTARGS(live_bytes_in_old));
+ }
+}
+
+void ShenandoahGenerationalFullGC::restore_top_before_promote(ShenandoahHeap* heap) {
+ for (size_t i = 0; i < heap->num_regions(); i++) {
+ ShenandoahHeapRegion* r = heap->get_region(i);
+ if (r->get_top_before_promote() != nullptr) {
+ r->restore_top_before_promote();
+ }
+ }
+}
+
+void ShenandoahGenerationalFullGC::account_for_region(ShenandoahHeapRegion* r, size_t ®ion_count, size_t ®ion_usage, size_t &humongous_waste) {
+ region_count++;
+ region_usage += r->used();
+ if (r->is_humongous_start()) {
+ // For each humongous object, we take this path once regardless of how many regions it spans.
+ HeapWord* obj_addr = r->bottom();
+ oop obj = cast_to_oop(obj_addr);
+ size_t word_size = obj->size();
+ size_t region_size_words = ShenandoahHeapRegion::region_size_words();
+ size_t overreach = word_size % region_size_words;
+ if (overreach != 0) {
+ humongous_waste += (region_size_words - overreach) * HeapWordSize;
+ }
+ // else, this humongous object aligns exactly on region size, so no waste.
+ }
+}
+
+void ShenandoahGenerationalFullGC::maybe_coalesce_and_fill_region(ShenandoahHeapRegion* r) {
+ if (r->is_pinned() && r->is_old() && r->is_active() && !r->is_humongous()) {
+ r->begin_preemptible_coalesce_and_fill();
+ r->oop_coalesce_and_fill(false);
+ }
+}
+
+void ShenandoahGenerationalFullGC::compute_balances() {
+ auto heap = ShenandoahGenerationalHeap::heap();
+
+ // In case this Full GC resulted from degeneration, clear the tally on anticipated promotion.
+ heap->old_generation()->set_promotion_potential(0);
+ // Invoke this in case we are able to transfer memory from OLD to YOUNG.
+ heap->compute_old_generation_balance(0, 0);
+}
+
+ShenandoahPrepareForGenerationalCompactionObjectClosure::ShenandoahPrepareForGenerationalCompactionObjectClosure(PreservedMarks* preserved_marks,
+ GrowableArray& empty_regions,
+ ShenandoahHeapRegion* from_region, uint worker_id) :
+ _preserved_marks(preserved_marks),
+ _heap(ShenandoahGenerationalHeap::heap()),
+ _tenuring_threshold(0),
+ _empty_regions(empty_regions),
+ _empty_regions_pos(0),
+ _old_to_region(nullptr),
+ _young_to_region(nullptr),
+ _from_region(nullptr),
+ _from_affiliation(ShenandoahAffiliation::FREE),
+ _old_compact_point(nullptr),
+ _young_compact_point(nullptr),
+ _worker_id(worker_id) {
+ assert(from_region != nullptr, "Worker needs from_region");
+ // assert from_region has live?
+ if (from_region->is_old()) {
+ _old_to_region = from_region;
+ _old_compact_point = from_region->bottom();
+ } else if (from_region->is_young()) {
+ _young_to_region = from_region;
+ _young_compact_point = from_region->bottom();
+ }
+
+ _tenuring_threshold = _heap->age_census()->tenuring_threshold();
+}
+
+void ShenandoahPrepareForGenerationalCompactionObjectClosure::set_from_region(ShenandoahHeapRegion* from_region) {
+ log_debug(gc)("Worker %u compacting %s Region " SIZE_FORMAT " which had used " SIZE_FORMAT " and %s live",
+ _worker_id, from_region->affiliation_name(),
+ from_region->index(), from_region->used(), from_region->has_live()? "has": "does not have");
+
+ _from_region = from_region;
+ _from_affiliation = from_region->affiliation();
+ if (_from_region->has_live()) {
+ if (_from_affiliation == ShenandoahAffiliation::OLD_GENERATION) {
+ if (_old_to_region == nullptr) {
+ _old_to_region = from_region;
+ _old_compact_point = from_region->bottom();
+ }
+ } else {
+ assert(_from_affiliation == ShenandoahAffiliation::YOUNG_GENERATION, "from_region must be OLD or YOUNG");
+ if (_young_to_region == nullptr) {
+ _young_to_region = from_region;
+ _young_compact_point = from_region->bottom();
+ }
+ }
+ } // else, we won't iterate over this _from_region so we don't need to set up to region to hold copies
+}
+
+void ShenandoahPrepareForGenerationalCompactionObjectClosure::finish() {
+ finish_old_region();
+ finish_young_region();
+}
+
+void ShenandoahPrepareForGenerationalCompactionObjectClosure::finish_old_region() {
+ if (_old_to_region != nullptr) {
+ log_debug(gc)("Planned compaction into Old Region " SIZE_FORMAT ", used: " SIZE_FORMAT " tabulated by worker %u",
+ _old_to_region->index(), _old_compact_point - _old_to_region->bottom(), _worker_id);
+ _old_to_region->set_new_top(_old_compact_point);
+ _old_to_region = nullptr;
+ }
+}
+
+void ShenandoahPrepareForGenerationalCompactionObjectClosure::finish_young_region() {
+ if (_young_to_region != nullptr) {
+ log_debug(gc)("Worker %u planned compaction into Young Region " SIZE_FORMAT ", used: " SIZE_FORMAT,
+ _worker_id, _young_to_region->index(), _young_compact_point - _young_to_region->bottom());
+ _young_to_region->set_new_top(_young_compact_point);
+ _young_to_region = nullptr;
+ }
+}
+
+bool ShenandoahPrepareForGenerationalCompactionObjectClosure::is_compact_same_region() {
+ return (_from_region == _old_to_region) || (_from_region == _young_to_region);
+}
+
+void ShenandoahPrepareForGenerationalCompactionObjectClosure::do_object(oop p) {
+ assert(_from_region != nullptr, "must set before work");
+ assert((_from_region->bottom() <= cast_from_oop(p)) && (cast_from_oop(p) < _from_region->top()),
+ "Object must reside in _from_region");
+ assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
+ assert(!_heap->complete_marking_context()->allocated_after_mark_start(p), "must be truly marked");
+
+ size_t obj_size = p->size();
+ uint from_region_age = _from_region->age();
+ uint object_age = p->age();
+
+ bool promote_object = false;
+ if ((_from_affiliation == ShenandoahAffiliation::YOUNG_GENERATION) &&
+ (from_region_age + object_age >= _tenuring_threshold)) {
+ if ((_old_to_region != nullptr) && (_old_compact_point + obj_size > _old_to_region->end())) {
+ finish_old_region();
+ _old_to_region = nullptr;
+ }
+ if (_old_to_region == nullptr) {
+ if (_empty_regions_pos < _empty_regions.length()) {
+ ShenandoahHeapRegion* new_to_region = _empty_regions.at(_empty_regions_pos);
+ _empty_regions_pos++;
+ new_to_region->set_affiliation(OLD_GENERATION);
+ _old_to_region = new_to_region;
+ _old_compact_point = _old_to_region->bottom();
+ promote_object = true;
+ }
+ // Else this worker thread does not yet have any empty regions into which this aged object can be promoted so
+ // we leave promote_object as false, deferring the promotion.
+ } else {
+ promote_object = true;
+ }
+ }
+
+ if (promote_object || (_from_affiliation == ShenandoahAffiliation::OLD_GENERATION)) {
+ assert(_old_to_region != nullptr, "_old_to_region should not be nullptr when evacuating to OLD region");
+ if (_old_compact_point + obj_size > _old_to_region->end()) {
+ ShenandoahHeapRegion* new_to_region;
+
+ log_debug(gc)("Worker %u finishing old region " SIZE_FORMAT ", compact_point: " PTR_FORMAT ", obj_size: " SIZE_FORMAT
+ ", &compact_point[obj_size]: " PTR_FORMAT ", region end: " PTR_FORMAT, _worker_id, _old_to_region->index(),
+ p2i(_old_compact_point), obj_size, p2i(_old_compact_point + obj_size), p2i(_old_to_region->end()));
+
+ // Object does not fit. Get a new _old_to_region.
+ finish_old_region();
+ if (_empty_regions_pos < _empty_regions.length()) {
+ new_to_region = _empty_regions.at(_empty_regions_pos);
+ _empty_regions_pos++;
+ new_to_region->set_affiliation(OLD_GENERATION);
+ } else {
+ // If we've exhausted the previously selected _old_to_region, we know that the _old_to_region is distinct
+ // from _from_region. That's because there is always room for _from_region to be compacted into itself.
+ // Since we're out of empty regions, let's use _from_region to hold the results of its own compaction.
+ new_to_region = _from_region;
+ }
+
+ assert(new_to_region != _old_to_region, "must not reuse same OLD to-region");
+ assert(new_to_region != nullptr, "must not be nullptr");
+ _old_to_region = new_to_region;
+ _old_compact_point = _old_to_region->bottom();
+ }
+
+ // Object fits into current region, record new location, if object does not move:
+ assert(_old_compact_point + obj_size <= _old_to_region->end(), "must fit");
+ shenandoah_assert_not_forwarded(nullptr, p);
+ if (_old_compact_point != cast_from_oop(p)) {
+ _preserved_marks->push_if_necessary(p, p->mark());
+ FullGCForwarding::forward_to(p, cast_to_oop(_old_compact_point));
+ }
+ _old_compact_point += obj_size;
+ } else {
+ assert(_from_affiliation == ShenandoahAffiliation::YOUNG_GENERATION,
+ "_from_region must be OLD_GENERATION or YOUNG_GENERATION");
+ assert(_young_to_region != nullptr, "_young_to_region should not be nullptr when compacting YOUNG _from_region");
+
+ // After full gc compaction, all regions have age 0. Embed the region's age into the object's age in order to preserve
+ // tenuring progress.
+ if (_heap->is_aging_cycle()) {
+ ShenandoahHeap::increase_object_age(p, from_region_age + 1);
+ } else {
+ ShenandoahHeap::increase_object_age(p, from_region_age);
+ }
+
+ if (_young_compact_point + obj_size > _young_to_region->end()) {
+ ShenandoahHeapRegion* new_to_region;
+
+ log_debug(gc)("Worker %u finishing young region " SIZE_FORMAT ", compact_point: " PTR_FORMAT ", obj_size: " SIZE_FORMAT
+ ", &compact_point[obj_size]: " PTR_FORMAT ", region end: " PTR_FORMAT, _worker_id, _young_to_region->index(),
+ p2i(_young_compact_point), obj_size, p2i(_young_compact_point + obj_size), p2i(_young_to_region->end()));
+
+ // Object does not fit. Get a new _young_to_region.
+ finish_young_region();
+ if (_empty_regions_pos < _empty_regions.length()) {
+ new_to_region = _empty_regions.at(_empty_regions_pos);
+ _empty_regions_pos++;
+ new_to_region->set_affiliation(YOUNG_GENERATION);
+ } else {
+ // If we've exhausted the previously selected _young_to_region, we know that the _young_to_region is distinct
+ // from _from_region. That's because there is always room for _from_region to be compacted into itself.
+ // Since we're out of empty regions, let's use _from_region to hold the results of its own compaction.
+ new_to_region = _from_region;
+ }
+
+ assert(new_to_region != _young_to_region, "must not reuse same OLD to-region");
+ assert(new_to_region != nullptr, "must not be nullptr");
+ _young_to_region = new_to_region;
+ _young_compact_point = _young_to_region->bottom();
+ }
+
+ // Object fits into current region, record new location, if object does not move:
+ assert(_young_compact_point + obj_size <= _young_to_region->end(), "must fit");
+ shenandoah_assert_not_forwarded(nullptr, p);
+
+ if (_young_compact_point != cast_from_oop(p)) {
+ _preserved_marks->push_if_necessary(p, p->mark());
+ FullGCForwarding::forward_to(p, cast_to_oop(_young_compact_point));
+ }
+ _young_compact_point += obj_size;
+ }
+}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.hpp
new file mode 100644
index 00000000000..d74bcefaaf2
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.hpp
@@ -0,0 +1,122 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONALFULLGC_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONALFULLGC_HPP
+
+#include "gc/shared/preservedMarks.hpp"
+#include "memory/iterator.hpp"
+#include "oops/oop.inline.hpp"
+#include "utilities/growableArray.hpp"
+
+class ShenandoahHeap;
+class ShenandoahHeapRegion;
+
+class ShenandoahGenerationalFullGC {
+public:
+ // Prepares the generational mode heap for a full collection.
+ static void prepare();
+
+ // Full GC may have compacted objects in the old generation, so we need to rebuild the card tables.
+ static void rebuild_remembered_set(ShenandoahHeap* heap);
+
+ // Records end of cycle for young and old and establishes size of live bytes in old
+ static void handle_completion(ShenandoahHeap* heap);
+
+ // Full GC may have promoted regions and may have temporarily violated constraints on the usage and
+ // capacity of the old generation. This method will balance the accounting of regions between the
+ // young and old generations. This is somewhat vestigial, but the outcome of this method is used
+ // when rebuilding the free sets.
+ static void balance_generations_after_gc(ShenandoahHeap* heap);
+
+ // This will compute the target size for the old generation. It will be expressed in terms of
+ // a region surplus and deficit, which will be redistributed accordingly after rebuilding the
+ // free set.
+ static void compute_balances();
+
+ // Rebuilding the free set may have resulted in regions being pulled in to the old generation
+ // evacuation reserve. For this reason, we must update the usage and capacity of the generations
+ // again. In the distant past, the free set did not know anything about generations, so we had
+ // a layer built above it to represent how much young/old memory was available. This layer is
+ // redundant and adds complexity. We would like to one day remove it. Until then, we must keep it
+ // synchronized with the free set's view of things.
+ static void balance_generations_after_rebuilding_free_set();
+
+ // Logs the number of live bytes marked in the old generation. This is _not_ the same
+ // value used as the baseline for the old generation _after_ the full gc is complete.
+ // The value reported in the logs does not include objects and regions that may be
+ // promoted during the full gc.
+ static void log_live_in_old(ShenandoahHeap* heap);
+
+ // This is used to tally the number, usage and space wasted by humongous objects for each generation.
+ static void account_for_region(ShenandoahHeapRegion* r, size_t ®ion_count, size_t ®ion_usage, size_t &humongous_waste);
+
+ // Regions which are scheduled for in-place promotion during evacuation temporarily
+ // have their top set to their end to prevent new objects from being allocated in them
+ // before they are promoted. If the full GC encounters such a region, it means the
+ // in-place promotion did not happen, and we must restore the original value of top.
+ static void restore_top_before_promote(ShenandoahHeap* heap);
+
+ // Pinned regions are not compacted, so they may still hold unmarked objects with
+ // references to reclaimed memory. Remembered set scanning will crash if it attempts
+ // to iterate the oops in these objects. This method fills in dead objects for pinned,
+ // old regions.
+ static void maybe_coalesce_and_fill_region(ShenandoahHeapRegion* r);
+};
+
+class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClosure {
+private:
+ PreservedMarks* const _preserved_marks;
+ ShenandoahGenerationalHeap* const _heap;
+ uint _tenuring_threshold;
+
+ // _empty_regions is a thread-local list of heap regions that have been completely emptied by this worker thread's
+ // compaction efforts. The worker thread that drives these efforts adds compacted regions to this list if the
+ // region has not been compacted onto itself.
+ GrowableArray& _empty_regions;
+ int _empty_regions_pos;
+ ShenandoahHeapRegion* _old_to_region;
+ ShenandoahHeapRegion* _young_to_region;
+ ShenandoahHeapRegion* _from_region;
+ ShenandoahAffiliation _from_affiliation;
+ HeapWord* _old_compact_point;
+ HeapWord* _young_compact_point;
+ uint _worker_id;
+
+public:
+ ShenandoahPrepareForGenerationalCompactionObjectClosure(PreservedMarks* preserved_marks,
+ GrowableArray& empty_regions,
+ ShenandoahHeapRegion* from_region, uint worker_id);
+
+ void set_from_region(ShenandoahHeapRegion* from_region);
+ void finish();
+ void finish_old_region();
+ void finish_young_region();
+ bool is_compact_same_region();
+ int empty_regions_pos() const { return _empty_regions_pos; }
+
+ void do_object(oop p) override;
+};
+
+#endif //SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONALFULLGC_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp
new file mode 100644
index 00000000000..5b8afc52b93
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp
@@ -0,0 +1,1140 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/shenandoah/shenandoahAgeCensus.hpp"
+#include "gc/shenandoah/shenandoahClosures.inline.hpp"
+#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
+#include "gc/shenandoah/shenandoahFreeSet.hpp"
+#include "gc/shenandoah/shenandoahGenerationalControlThread.hpp"
+#include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
+#include "gc/shenandoah/shenandoahInitLogger.hpp"
+#include "gc/shenandoah/shenandoahMemoryPool.hpp"
+#include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
+#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
+#include "gc/shenandoah/shenandoahRegulatorThread.hpp"
+#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
+#include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
+#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
+#include "gc/shenandoah/shenandoahUtils.hpp"
+#include "logging/log.hpp"
+#include "utilities/events.hpp"
+
+
+class ShenandoahGenerationalInitLogger : public ShenandoahInitLogger {
+public:
+ static void print() {
+ ShenandoahGenerationalInitLogger logger;
+ logger.print_all();
+ }
+
+ void print_heap() override {
+ ShenandoahInitLogger::print_heap();
+
+ ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
+
+ ShenandoahYoungGeneration* young = heap->young_generation();
+ log_info(gc, init)("Young Generation Soft Size: " EXACTFMT, EXACTFMTARGS(young->soft_max_capacity()));
+ log_info(gc, init)("Young Generation Max: " EXACTFMT, EXACTFMTARGS(young->max_capacity()));
+
+ ShenandoahOldGeneration* old = heap->old_generation();
+ log_info(gc, init)("Old Generation Soft Size: " EXACTFMT, EXACTFMTARGS(old->soft_max_capacity()));
+ log_info(gc, init)("Old Generation Max: " EXACTFMT, EXACTFMTARGS(old->max_capacity()));
+ }
+
+protected:
+ void print_gc_specific() override {
+ ShenandoahInitLogger::print_gc_specific();
+
+ ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
+ log_info(gc, init)("Young Heuristics: %s", heap->young_generation()->heuristics()->name());
+ log_info(gc, init)("Old Heuristics: %s", heap->old_generation()->heuristics()->name());
+ }
+};
+
+size_t ShenandoahGenerationalHeap::calculate_min_plab() {
+ return align_up(PLAB::min_size(), CardTable::card_size_in_words());
+}
+
+size_t ShenandoahGenerationalHeap::calculate_max_plab() {
+ size_t MaxTLABSizeWords = ShenandoahHeapRegion::max_tlab_size_words();
+ return align_down(MaxTLABSizeWords, CardTable::card_size_in_words());
+}
+
+// Returns size in bytes
+size_t ShenandoahGenerationalHeap::unsafe_max_tlab_alloc(Thread *thread) const {
+ return MIN2(ShenandoahHeapRegion::max_tlab_size_bytes(), young_generation()->available());
+}
+
+ShenandoahGenerationalHeap::ShenandoahGenerationalHeap(ShenandoahCollectorPolicy* policy) :
+ ShenandoahHeap(policy),
+ _age_census(nullptr),
+ _evac_tracker(new ShenandoahEvacuationTracker()),
+ _min_plab_size(calculate_min_plab()),
+ _max_plab_size(calculate_max_plab()),
+ _regulator_thread(nullptr),
+ _young_gen_memory_pool(nullptr),
+ _old_gen_memory_pool(nullptr) {
+ assert(is_aligned(_min_plab_size, CardTable::card_size_in_words()), "min_plab_size must be aligned");
+ assert(is_aligned(_max_plab_size, CardTable::card_size_in_words()), "max_plab_size must be aligned");
+}
+
+void ShenandoahGenerationalHeap::post_initialize() {
+ ShenandoahHeap::post_initialize();
+ _age_census = new ShenandoahAgeCensus();
+}
+
+void ShenandoahGenerationalHeap::print_init_logger() const {
+ ShenandoahGenerationalInitLogger logger;
+ logger.print_all();
+}
+
+void ShenandoahGenerationalHeap::print_tracing_info() const {
+ ShenandoahHeap::print_tracing_info();
+
+ LogTarget(Info, gc, stats) lt;
+ if (lt.is_enabled()) {
+ LogStream ls(lt);
+ ls.cr();
+ ls.cr();
+ evac_tracker()->print_global_on(&ls);
+ }
+}
+
+void ShenandoahGenerationalHeap::initialize_heuristics() {
+ // Initialize global generation and heuristics even in generational mode.
+ ShenandoahHeap::initialize_heuristics();
+
+ // Max capacity is the maximum _allowed_ capacity. That is, the maximum allowed capacity
+ // for old would be total heap - minimum capacity of young. This means the sum of the maximum
+ // allowed for old and young could exceed the total heap size. It remains the case that the
+ // _actual_ capacity of young + old = total.
+ _generation_sizer.heap_size_changed(max_capacity());
+ size_t initial_capacity_young = _generation_sizer.max_young_size();
+ size_t max_capacity_young = _generation_sizer.max_young_size();
+ size_t initial_capacity_old = max_capacity() - max_capacity_young;
+ size_t max_capacity_old = max_capacity() - initial_capacity_young;
+
+ _young_generation = new ShenandoahYoungGeneration(max_workers(), max_capacity_young, initial_capacity_young);
+ _old_generation = new ShenandoahOldGeneration(max_workers(), max_capacity_old, initial_capacity_old);
+ _young_generation->initialize_heuristics(mode());
+ _old_generation->initialize_heuristics(mode());
+}
+
+void ShenandoahGenerationalHeap::initialize_serviceability() {
+ assert(mode()->is_generational(), "Only for the generational mode");
+ _young_gen_memory_pool = new ShenandoahYoungGenMemoryPool(this);
+ _old_gen_memory_pool = new ShenandoahOldGenMemoryPool(this);
+ cycle_memory_manager()->add_pool(_young_gen_memory_pool);
+ cycle_memory_manager()->add_pool(_old_gen_memory_pool);
+ stw_memory_manager()->add_pool(_young_gen_memory_pool);
+ stw_memory_manager()->add_pool(_old_gen_memory_pool);
+}
+
+GrowableArray ShenandoahGenerationalHeap::memory_pools() {
+ assert(mode()->is_generational(), "Only for the generational mode");
+ GrowableArray memory_pools(2);
+ memory_pools.append(_young_gen_memory_pool);
+ memory_pools.append(_old_gen_memory_pool);
+ return memory_pools;
+}
+
+void ShenandoahGenerationalHeap::initialize_controller() {
+ auto control_thread = new ShenandoahGenerationalControlThread();
+ _control_thread = control_thread;
+ _regulator_thread = new ShenandoahRegulatorThread(control_thread);
+}
+
+void ShenandoahGenerationalHeap::gc_threads_do(ThreadClosure* tcl) const {
+ if (!shenandoah_policy()->is_at_shutdown()) {
+ ShenandoahHeap::gc_threads_do(tcl);
+ tcl->do_thread(regulator_thread());
+ }
+}
+
+void ShenandoahGenerationalHeap::stop() {
+ regulator_thread()->stop();
+ ShenandoahHeap::stop();
+}
+
+void ShenandoahGenerationalHeap::evacuate_collection_set(bool concurrent) {
+ ShenandoahRegionIterator regions;
+ ShenandoahGenerationalEvacuationTask task(this, ®ions, concurrent, false /* only promote regions */);
+ workers()->run_task(&task);
+}
+
+void ShenandoahGenerationalHeap::promote_regions_in_place(bool concurrent) {
+ ShenandoahRegionIterator regions;
+ ShenandoahGenerationalEvacuationTask task(this, ®ions, concurrent, true /* only promote regions */);
+ workers()->run_task(&task);
+}
+
+oop ShenandoahGenerationalHeap::evacuate_object(oop p, Thread* thread) {
+ assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
+ if (ShenandoahThreadLocalData::is_oom_during_evac(thread)) {
+ // This thread went through the OOM during evac protocol and it is safe to return
+ // the forward pointer. It must not attempt to evacuate anymore.
+ return ShenandoahBarrierSet::resolve_forwarded(p);
+ }
+
+ assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
+
+ ShenandoahHeapRegion* r = heap_region_containing(p);
+ assert(!r->is_humongous(), "never evacuate humongous objects");
+
+ ShenandoahAffiliation target_gen = r->affiliation();
+ // gc_generation() can change asynchronously and should not be used here.
+ assert(active_generation() != nullptr, "Error");
+ if (active_generation()->is_young() && target_gen == YOUNG_GENERATION) {
+ markWord mark = p->mark();
+ if (mark.is_marked()) {
+ // Already forwarded.
+ return ShenandoahBarrierSet::resolve_forwarded(p);
+ }
+
+ if (mark.has_displaced_mark_helper()) {
+ // We don't want to deal with MT here just to ensure we read the right mark word.
+ // Skip the potential promotion attempt for this one.
+ } else if (r->age() + mark.age() >= age_census()->tenuring_threshold()) {
+ oop result = try_evacuate_object(p, thread, r, OLD_GENERATION);
+ if (result != nullptr) {
+ return result;
+ }
+ // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen.
+ }
+ }
+ return try_evacuate_object(p, thread, r, target_gen);
+}
+
+// try_evacuate_object registers the object and dirties the associated remembered set information when evacuating
+// to OLD_GENERATION.
+oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
+ ShenandoahAffiliation target_gen) {
+ bool alloc_from_lab = true;
+ bool has_plab = false;
+ HeapWord* copy = nullptr;
+ size_t size = ShenandoahForwarding::size(p);
+ bool is_promotion = (target_gen == OLD_GENERATION) && from_region->is_young();
+
+#ifdef ASSERT
+ if (ShenandoahOOMDuringEvacALot &&
+ (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
+ copy = nullptr;
+ } else {
+#endif
+ if (UseTLAB) {
+ switch (target_gen) {
+ case YOUNG_GENERATION: {
+ copy = allocate_from_gclab(thread, size);
+ if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
+ // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting
+ // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
+ ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
+ copy = allocate_from_gclab(thread, size);
+ // If we still get nullptr, we'll try a shared allocation below.
+ }
+ break;
+ }
+ case OLD_GENERATION: {
+ PLAB* plab = ShenandoahThreadLocalData::plab(thread);
+ if (plab != nullptr) {
+ has_plab = true;
+ copy = allocate_from_plab(thread, size, is_promotion);
+ if ((copy == nullptr) && (size < ShenandoahThreadLocalData::plab_size(thread)) &&
+ ShenandoahThreadLocalData::plab_retries_enabled(thread)) {
+ // PLAB allocation failed because we are bumping up against the limit on old evacuation reserve or because
+ // the requested object does not fit within the current plab but the plab still has an "abundance" of memory,
+ // where abundance is defined as >= ShenGenHeap::plab_min_size(). In the former case, we try shrinking the
+ // desired PLAB size to the minimum and retry PLAB allocation to avoid cascading of shared memory allocations.
+ if (plab->words_remaining() < plab_min_size()) {
+ ShenandoahThreadLocalData::set_plab_size(thread, plab_min_size());
+ copy = allocate_from_plab(thread, size, is_promotion);
+ // If we still get nullptr, we'll try a shared allocation below.
+ if (copy == nullptr) {
+ // If retry fails, don't continue to retry until we have success (probably in next GC pass)
+ ShenandoahThreadLocalData::disable_plab_retries(thread);
+ }
+ }
+ // else, copy still equals nullptr. this causes shared allocation below, preserving this plab for future needs.
+ }
+ }
+ break;
+ }
+ default: {
+ ShouldNotReachHere();
+ break;
+ }
+ }
+ }
+
+ if (copy == nullptr) {
+ // If we failed to allocate in LAB, we'll try a shared allocation.
+ if (!is_promotion || !has_plab || (size > PLAB::min_size())) {
+ ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen, is_promotion);
+ copy = allocate_memory(req);
+ alloc_from_lab = false;
+ }
+ // else, we leave copy equal to nullptr, signaling a promotion failure below if appropriate.
+ // We choose not to promote objects smaller than PLAB::min_size() by way of shared allocations, as this is too
+ // costly. Instead, we'll simply "evacuate" to young-gen memory (using a GCLAB) and will promote in a future
+ // evacuation pass. This condition is denoted by: is_promotion && has_plab && (size <= PLAB::min_size())
+ }
+#ifdef ASSERT
+ }
+#endif
+
+ if (copy == nullptr) {
+ if (target_gen == OLD_GENERATION) {
+ if (from_region->is_young()) {
+ // Signal that promotion failed. Will evacuate this old object somewhere in young gen.
+ old_generation()->handle_failed_promotion(thread, size);
+ return nullptr;
+ } else {
+ // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this
+ // after the evacuation threads have finished.
+ old_generation()->handle_failed_evacuation();
+ }
+ }
+
+ control_thread()->handle_alloc_failure_evac(size);
+
+ oom_evac_handler()->handle_out_of_memory_during_evacuation();
+
+ return ShenandoahBarrierSet::resolve_forwarded(p);
+ }
+
+ // Copy the object:
+ NOT_PRODUCT(evac_tracker()->begin_evacuation(thread, size * HeapWordSize));
+ Copy::aligned_disjoint_words(cast_from_oop(p), copy, size);
+ oop copy_val = cast_to_oop(copy);
+
+ // Update the age of the evacuated object
+ if (target_gen == YOUNG_GENERATION && is_aging_cycle()) {
+ ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1);
+ }
+
+ // Try to install the new forwarding pointer.
+ oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
+ if (result == copy_val) {
+ // Successfully evacuated. Our copy is now the public one!
+
+ // This is necessary for virtual thread support. This uses the mark word without
+ // considering that it may now be a forwarding pointer (and could therefore crash).
+ // Secondarily, we do not want to spend cycles relativizing stack chunks for oops
+ // that lost the evacuation race (and will therefore not become visible). It is
+ // safe to do this on the public copy (this is also done during concurrent mark).
+ ContinuationGCSupport::relativize_stack_chunk(copy_val);
+
+ // Record that the evacuation succeeded
+ NOT_PRODUCT(evac_tracker()->end_evacuation(thread, size * HeapWordSize));
+
+ if (target_gen == OLD_GENERATION) {
+ old_generation()->handle_evacuation(copy, size, from_region->is_young());
+ } else {
+ // When copying to the old generation above, we don't care
+ // about recording object age in the census stats.
+ assert(target_gen == YOUNG_GENERATION, "Error");
+ // We record this census only when simulating pre-adaptive tenuring behavior, or
+ // when we have been asked to record the census at evacuation rather than at mark
+ if (ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring) {
+ evac_tracker()->record_age(thread, size * HeapWordSize, ShenandoahHeap::get_object_age(copy_val));
+ }
+ }
+ shenandoah_assert_correct(nullptr, copy_val);
+ return copy_val;
+ } else {
+ // Failed to evacuate. We need to deal with the object that is left behind. Since this
+ // new allocation is certainly after TAMS, it will be considered live in the next cycle.
+ // But if it happens to contain references to evacuated regions, those references would
+ // not get updated for this stale copy during this cycle, and we will crash while scanning
+ // it the next cycle.
+ if (alloc_from_lab) {
+ // For LAB allocations, it is enough to rollback the allocation ptr. Either the next
+ // object will overwrite this stale copy, or the filler object on LAB retirement will
+ // do this.
+ switch (target_gen) {
+ case YOUNG_GENERATION: {
+ ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size);
+ break;
+ }
+ case OLD_GENERATION: {
+ ShenandoahThreadLocalData::plab(thread)->undo_allocation(copy, size);
+ if (is_promotion) {
+ ShenandoahThreadLocalData::subtract_from_plab_promoted(thread, size * HeapWordSize);
+ }
+ break;
+ }
+ default: {
+ ShouldNotReachHere();
+ break;
+ }
+ }
+ } else {
+ // For non-LAB allocations, we have no way to retract the allocation, and
+ // have to explicitly overwrite the copy with the filler object. With that overwrite,
+ // we have to keep the fwdptr initialized and pointing to our (stale) copy.
+ assert(size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size");
+ fill_with_object(copy, size);
+ shenandoah_assert_correct(nullptr, copy_val);
+ // For non-LAB allocations, the object has already been registered
+ }
+ shenandoah_assert_correct(nullptr, result);
+ return result;
+ }
+}
+
+inline HeapWord* ShenandoahGenerationalHeap::allocate_from_plab(Thread* thread, size_t size, bool is_promotion) {
+ assert(UseTLAB, "TLABs should be enabled");
+
+ PLAB* plab = ShenandoahThreadLocalData::plab(thread);
+ HeapWord* obj;
+
+ if (plab == nullptr) {
+ assert(!thread->is_Java_thread() && !thread->is_Worker_thread(), "Performance: thread should have PLAB: %s", thread->name());
+ // No PLABs in this thread, fallback to shared allocation
+ return nullptr;
+ } else if (is_promotion && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) {
+ return nullptr;
+ }
+ // if plab->word_size() <= 0, thread's plab not yet initialized for this pass, so allow_plab_promotions() is not trustworthy
+ obj = plab->allocate(size);
+ if ((obj == nullptr) && (plab->words_remaining() < plab_min_size())) {
+ // allocate_from_plab_slow will establish allow_plab_promotions(thread) for future invocations
+ obj = allocate_from_plab_slow(thread, size, is_promotion);
+ }
+ // if plab->words_remaining() >= ShenGenHeap::heap()->plab_min_size(), just return nullptr so we can use a shared allocation
+ if (obj == nullptr) {
+ return nullptr;
+ }
+
+ if (is_promotion) {
+ ShenandoahThreadLocalData::add_to_plab_promoted(thread, size * HeapWordSize);
+ }
+ return obj;
+}
+
+// Establish a new PLAB and allocate size HeapWords within it.
+HeapWord* ShenandoahGenerationalHeap::allocate_from_plab_slow(Thread* thread, size_t size, bool is_promotion) {
+ // New object should fit the PLAB size
+
+ assert(mode()->is_generational(), "PLABs only relevant to generational GC");
+ const size_t plab_min_size = this->plab_min_size();
+ // PLABs are aligned to card boundaries to avoid synchronization with concurrent
+ // allocations in other PLABs.
+ const size_t min_size = (size > plab_min_size)? align_up(size, CardTable::card_size_in_words()): plab_min_size;
+
+ // Figure out size of new PLAB, using value determined at last refill.
+ size_t cur_size = ShenandoahThreadLocalData::plab_size(thread);
+ if (cur_size == 0) {
+ cur_size = plab_min_size;
+ }
+
+ // Expand aggressively, doubling at each refill in this epoch, ceiling at plab_max_size()
+ size_t future_size = MIN2(cur_size * 2, plab_max_size());
+ // Doubling, starting at a card-multiple, should give us a card-multiple. (Ceiling and floor
+ // are card multiples.)
+ assert(is_aligned(future_size, CardTable::card_size_in_words()), "Card multiple by construction, future_size: " SIZE_FORMAT
+ ", card_size: " SIZE_FORMAT ", cur_size: " SIZE_FORMAT ", max: " SIZE_FORMAT,
+ future_size, (size_t) CardTable::card_size_in_words(), cur_size, plab_max_size());
+
+ // Record new heuristic value even if we take any shortcut. This captures
+ // the case when moderately-sized objects always take a shortcut. At some point,
+ // heuristics should catch up with them. Note that the requested cur_size may
+ // not be honored, but we remember that this is the preferred size.
+ log_debug(gc, free)("Set new PLAB size: " SIZE_FORMAT, future_size);
+ ShenandoahThreadLocalData::set_plab_size(thread, future_size);
+ if (cur_size < size) {
+ // The PLAB to be allocated is still not large enough to hold the object. Fall back to shared allocation.
+ // This avoids retiring perfectly good PLABs in order to represent a single large object allocation.
+ log_debug(gc, free)("Current PLAB size (" SIZE_FORMAT ") is too small for " SIZE_FORMAT, cur_size, size);
+ return nullptr;
+ }
+
+ // Retire current PLAB, and allocate a new one.
+ PLAB* plab = ShenandoahThreadLocalData::plab(thread);
+ if (plab->words_remaining() < plab_min_size) {
+ // Retire current PLAB. This takes care of any PLAB book-keeping.
+ // retire_plab() registers the remnant filler object with the remembered set scanner without a lock.
+ // Since PLABs are card-aligned, concurrent registrations in other PLABs don't interfere.
+ retire_plab(plab, thread);
+
+ size_t actual_size = 0;
+ HeapWord* plab_buf = allocate_new_plab(min_size, cur_size, &actual_size);
+ if (plab_buf == nullptr) {
+ if (min_size == plab_min_size) {
+ // Disable PLAB promotions for this thread because we cannot even allocate a minimal PLAB. This allows us
+ // to fail faster on subsequent promotion attempts.
+ ShenandoahThreadLocalData::disable_plab_promotions(thread);
+ }
+ return nullptr;
+ } else {
+ ShenandoahThreadLocalData::enable_plab_retries(thread);
+ }
+ // Since the allocated PLAB may have been down-sized for alignment, plab->allocate(size) below may still fail.
+ if (ZeroTLAB) {
+ // ... and clear it.
+ Copy::zero_to_words(plab_buf, actual_size);
+ } else {
+ // ...and zap just allocated object.
+#ifdef ASSERT
+ // Skip mangling the space corresponding to the object header to
+ // ensure that the returned space is not considered parsable by
+ // any concurrent GC thread.
+ size_t hdr_size = oopDesc::header_size();
+ Copy::fill_to_words(plab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
+#endif // ASSERT
+ }
+ assert(is_aligned(actual_size, CardTable::card_size_in_words()), "Align by design");
+ plab->set_buf(plab_buf, actual_size);
+ if (is_promotion && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) {
+ return nullptr;
+ }
+ return plab->allocate(size);
+ } else {
+ // If there's still at least min_size() words available within the current plab, don't retire it. Let's nibble
+ // away on this plab as long as we can. Meanwhile, return nullptr to force this particular allocation request
+ // to be satisfied with a shared allocation. By packing more promotions into the previously allocated PLAB, we
+ // reduce the likelihood of evacuation failures, and we reduce the need for downsizing our PLABs.
+ return nullptr;
+ }
+}
+
+HeapWord* ShenandoahGenerationalHeap::allocate_new_plab(size_t min_size, size_t word_size, size_t* actual_size) {
+ // Align requested sizes to card-sized multiples. Align down so that we don't violate max size of TLAB.
+ assert(is_aligned(min_size, CardTable::card_size_in_words()), "Align by design");
+ assert(word_size >= min_size, "Requested PLAB is too small");
+
+ ShenandoahAllocRequest req = ShenandoahAllocRequest::for_plab(min_size, word_size);
+ // Note that allocate_memory() sets a thread-local flag to prohibit further promotions by this thread
+ // if we are at risk of infringing on the old-gen evacuation budget.
+ HeapWord* res = allocate_memory(req);
+ if (res != nullptr) {
+ *actual_size = req.actual_size();
+ } else {
+ *actual_size = 0;
+ }
+ assert(is_aligned(res, CardTable::card_size_in_words()), "Align by design");
+ return res;
+}
+
+void ShenandoahGenerationalHeap::retire_plab(PLAB* plab, Thread* thread) {
+ // We don't enforce limits on plab evacuations. We let it consume all available old-gen memory in order to reduce
+ // probability of an evacuation failure. We do enforce limits on promotion, to make sure that excessive promotion
+ // does not result in an old-gen evacuation failure. Note that a failed promotion is relatively harmless. Any
+ // object that fails to promote in the current cycle will be eligible for promotion in a subsequent cycle.
+
+ // When the plab was instantiated, its entirety was treated as if the entire buffer was going to be dedicated to
+ // promotions. Now that we are retiring the buffer, we adjust for the reality that the plab is not entirely promotions.
+ // 1. Some of the plab may have been dedicated to evacuations.
+ // 2. Some of the plab may have been abandoned due to waste (at the end of the plab).
+ size_t not_promoted =
+ ShenandoahThreadLocalData::get_plab_actual_size(thread) - ShenandoahThreadLocalData::get_plab_promoted(thread);
+ ShenandoahThreadLocalData::reset_plab_promoted(thread);
+ ShenandoahThreadLocalData::set_plab_actual_size(thread, 0);
+ if (not_promoted > 0) {
+ old_generation()->unexpend_promoted(not_promoted);
+ }
+ const size_t original_waste = plab->waste();
+ HeapWord* const top = plab->top();
+
+ // plab->retire() overwrites unused memory between plab->top() and plab->hard_end() with a dummy object to make memory parsable.
+ // It adds the size of this unused memory, in words, to plab->waste().
+ plab->retire();
+ if (top != nullptr && plab->waste() > original_waste && is_in_old(top)) {
+ // If retiring the plab created a filler object, then we need to register it with our card scanner so it can
+ // safely walk the region backing the plab.
+ log_debug(gc)("retire_plab() is registering remnant of size " SIZE_FORMAT " at " PTR_FORMAT,
+ plab->waste() - original_waste, p2i(top));
+ // No lock is necessary because the PLAB memory is aligned on card boundaries.
+ old_generation()->card_scan()->register_object_without_lock(top);
+ }
+}
+
+void ShenandoahGenerationalHeap::retire_plab(PLAB* plab) {
+ Thread* thread = Thread::current();
+ retire_plab(plab, thread);
+}
+
+ShenandoahGenerationalHeap::TransferResult ShenandoahGenerationalHeap::balance_generations() {
+ shenandoah_assert_heaplocked_or_safepoint();
+
+ ShenandoahOldGeneration* old_gen = old_generation();
+ const ssize_t old_region_balance = old_gen->get_region_balance();
+ old_gen->set_region_balance(0);
+
+ if (old_region_balance > 0) {
+ const auto old_region_surplus = checked_cast(old_region_balance);
+ const bool success = generation_sizer()->transfer_to_young(old_region_surplus);
+ return TransferResult {
+ success, old_region_surplus, "young"
+ };
+ }
+
+ if (old_region_balance < 0) {
+ const auto old_region_deficit = checked_cast(-old_region_balance);
+ const bool success = generation_sizer()->transfer_to_old(old_region_deficit);
+ if (!success) {
+ old_gen->handle_failed_transfer();
+ }
+ return TransferResult {
+ success, old_region_deficit, "old"
+ };
+ }
+
+ return TransferResult {true, 0, "none"};
+}
+
+// Make sure old-generation is large enough, but no larger than is necessary, to hold mixed evacuations
+// and promotions, if we anticipate either. Any deficit is provided by the young generation, subject to
+// xfer_limit, and any surplus is transferred to the young generation.
+// xfer_limit is the maximum we're able to transfer from young to old.
+void ShenandoahGenerationalHeap::compute_old_generation_balance(size_t old_xfer_limit, size_t old_cset_regions) {
+
+ // We can limit the old reserve to the size of anticipated promotions:
+ // max_old_reserve is an upper bound on memory evacuated from old and promoted to old,
+ // clamped by the old generation space available.
+ //
+ // Here's the algebra.
+ // Let SOEP = ShenandoahOldEvacRatioPercent,
+ // OE = old evac,
+ // YE = young evac, and
+ // TE = total evac = OE + YE
+ // By definition:
+ // SOEP/100 = OE/TE
+ // = OE/(OE+YE)
+ // => SOEP/(100-SOEP) = OE/((OE+YE)-OE) // componendo-dividendo: If a/b = c/d, then a/(b-a) = c/(d-c)
+ // = OE/YE
+ // => OE = YE*SOEP/(100-SOEP)
+
+ // We have to be careful in the event that SOEP is set to 100 by the user.
+ assert(ShenandoahOldEvacRatioPercent <= 100, "Error");
+ const size_t old_available = old_generation()->available();
+ // The free set will reserve this amount of memory to hold young evacuations
+ const size_t young_reserve = (young_generation()->max_capacity() * ShenandoahEvacReserve) / 100;
+
+ // In the case that ShenandoahOldEvacRatioPercent equals 100, max_old_reserve is limited only by xfer_limit.
+
+ const double bound_on_old_reserve = old_available + old_xfer_limit + young_reserve;
+ const double max_old_reserve = (ShenandoahOldEvacRatioPercent == 100)?
+ bound_on_old_reserve: MIN2(double(young_reserve * ShenandoahOldEvacRatioPercent) / double(100 - ShenandoahOldEvacRatioPercent),
+ bound_on_old_reserve);
+
+ const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
+
+ // Decide how much old space we should reserve for a mixed collection
+ double reserve_for_mixed = 0;
+ if (old_generation()->has_unprocessed_collection_candidates()) {
+ // We want this much memory to be unfragmented in order to reliably evacuate old. This is conservative because we
+ // may not evacuate the entirety of unprocessed candidates in a single mixed evacuation.
+ const double max_evac_need = (double(old_generation()->unprocessed_collection_candidates_live_memory()) * ShenandoahOldEvacWaste);
+ assert(old_available >= old_generation()->free_unaffiliated_regions() * region_size_bytes,
+ "Unaffiliated available must be less than total available");
+ const double old_fragmented_available = double(old_available - old_generation()->free_unaffiliated_regions() * region_size_bytes);
+ reserve_for_mixed = max_evac_need + old_fragmented_available;
+ if (reserve_for_mixed > max_old_reserve) {
+ reserve_for_mixed = max_old_reserve;
+ }
+ }
+
+ // Decide how much space we should reserve for promotions from young
+ size_t reserve_for_promo = 0;
+ const size_t promo_load = old_generation()->get_promotion_potential();
+ const bool doing_promotions = promo_load > 0;
+ if (doing_promotions) {
+ // We're promoting and have a bound on the maximum amount that can be promoted
+ assert(max_old_reserve >= reserve_for_mixed, "Sanity");
+ const size_t available_for_promotions = max_old_reserve - reserve_for_mixed;
+ reserve_for_promo = MIN2((size_t)(promo_load * ShenandoahPromoEvacWaste), available_for_promotions);
+ }
+
+ // This is the total old we want to ideally reserve
+ const size_t old_reserve = reserve_for_mixed + reserve_for_promo;
+ assert(old_reserve <= max_old_reserve, "cannot reserve more than max for old evacuations");
+
+ // We now check if the old generation is running a surplus or a deficit.
+ const size_t max_old_available = old_generation()->available() + old_cset_regions * region_size_bytes;
+ if (max_old_available >= old_reserve) {
+ // We are running a surplus, so the old region surplus can go to young
+ const size_t old_surplus = (max_old_available - old_reserve) / region_size_bytes;
+ const size_t unaffiliated_old_regions = old_generation()->free_unaffiliated_regions() + old_cset_regions;
+ const size_t old_region_surplus = MIN2(old_surplus, unaffiliated_old_regions);
+ old_generation()->set_region_balance(checked_cast(old_region_surplus));
+ } else {
+ // We are running a deficit which we'd like to fill from young.
+ // Ignore that this will directly impact young_generation()->max_capacity(),
+ // indirectly impacting young_reserve and old_reserve. These computations are conservative.
+ // Note that deficit is rounded up by one region.
+ const size_t old_need = (old_reserve - max_old_available + region_size_bytes - 1) / region_size_bytes;
+ const size_t max_old_region_xfer = old_xfer_limit / region_size_bytes;
+
+ // Round down the regions we can transfer from young to old. If we're running short
+ // on young-gen memory, we restrict the xfer. Old-gen collection activities will be
+ // curtailed if the budget is restricted.
+ const size_t old_region_deficit = MIN2(old_need, max_old_region_xfer);
+ old_generation()->set_region_balance(0 - checked_cast(old_region_deficit));
+ }
+}
+
+void ShenandoahGenerationalHeap::reset_generation_reserves() {
+ young_generation()->set_evacuation_reserve(0);
+ old_generation()->set_evacuation_reserve(0);
+ old_generation()->set_promoted_reserve(0);
+}
+
+void ShenandoahGenerationalHeap::TransferResult::print_on(const char* when, outputStream* ss) const {
+ auto heap = ShenandoahGenerationalHeap::heap();
+ ShenandoahYoungGeneration* const young_gen = heap->young_generation();
+ ShenandoahOldGeneration* const old_gen = heap->old_generation();
+ const size_t young_available = young_gen->available();
+ const size_t old_available = old_gen->available();
+ ss->print_cr("After %s, %s " SIZE_FORMAT " regions to %s to prepare for next gc, old available: "
+ PROPERFMT ", young_available: " PROPERFMT,
+ when,
+ success? "successfully transferred": "failed to transfer", region_count, region_destination,
+ PROPERFMTARGS(old_available), PROPERFMTARGS(young_available));
+}
+
+void ShenandoahGenerationalHeap::coalesce_and_fill_old_regions(bool concurrent) {
+ class ShenandoahGlobalCoalesceAndFill : public WorkerTask {
+ private:
+ ShenandoahPhaseTimings::Phase _phase;
+ ShenandoahRegionIterator _regions;
+ public:
+ explicit ShenandoahGlobalCoalesceAndFill(ShenandoahPhaseTimings::Phase phase) :
+ WorkerTask("Shenandoah Global Coalesce"),
+ _phase(phase) {}
+
+ void work(uint worker_id) override {
+ ShenandoahWorkerTimingsTracker timer(_phase,
+ ShenandoahPhaseTimings::ScanClusters,
+ worker_id, true);
+ ShenandoahHeapRegion* region;
+ while ((region = _regions.next()) != nullptr) {
+ // old region is not in the collection set and was not immediately trashed
+ if (region->is_old() && region->is_active() && !region->is_humongous()) {
+ // Reset the coalesce and fill boundary because this is a global collect
+ // and cannot be preempted by young collects. We want to be sure the entire
+ // region is coalesced here and does not resume from a previously interrupted
+ // or completed coalescing.
+ region->begin_preemptible_coalesce_and_fill();
+ region->oop_coalesce_and_fill(false);
+ }
+ }
+ }
+ };
+
+ ShenandoahPhaseTimings::Phase phase = concurrent ?
+ ShenandoahPhaseTimings::conc_coalesce_and_fill :
+ ShenandoahPhaseTimings::degen_gc_coalesce_and_fill;
+
+ // This is not cancellable
+ ShenandoahGlobalCoalesceAndFill coalesce(phase);
+ workers()->run_task(&coalesce);
+ old_generation()->set_parsable(true);
+}
+
+template
+class ShenandoahGenerationalUpdateHeapRefsTask : public WorkerTask {
+private:
+ ShenandoahGenerationalHeap* _heap;
+ ShenandoahRegionIterator* _regions;
+ ShenandoahRegionChunkIterator* _work_chunks;
+
+public:
+ explicit ShenandoahGenerationalUpdateHeapRefsTask(ShenandoahRegionIterator* regions,
+ ShenandoahRegionChunkIterator* work_chunks) :
+ WorkerTask("Shenandoah Update References"),
+ _heap(ShenandoahGenerationalHeap::heap()),
+ _regions(regions),
+ _work_chunks(work_chunks)
+ {
+ bool old_bitmap_stable = _heap->old_generation()->is_mark_complete();
+ log_debug(gc, remset)("Update refs, scan remembered set using bitmap: %s", BOOL_TO_STR(old_bitmap_stable));
+ }
+
+ void work(uint worker_id) {
+ if (CONCURRENT) {
+ ShenandoahConcurrentWorkerSession worker_session(worker_id);
+ ShenandoahSuspendibleThreadSetJoiner stsj;
+ do_work(worker_id);
+ } else {
+ ShenandoahParallelWorkerSession worker_session(worker_id);
+ do_work(worker_id);
+ }
+ }
+
+private:
+ template
+ void do_work(uint worker_id) {
+ T cl;
+
+ if (CONCURRENT && (worker_id == 0)) {
+ // We ask the first worker to replenish the Mutator free set by moving regions previously reserved to hold the
+ // results of evacuation. These reserves are no longer necessary because evacuation has completed.
+ size_t cset_regions = _heap->collection_set()->count();
+
+ // Now that evacuation is done, we can reassign any regions that had been reserved to hold the results of evacuation
+ // to the mutator free set. At the end of GC, we will have cset_regions newly evacuated fully empty regions from
+ // which we will be able to replenish the Collector free set and the OldCollector free set in preparation for the
+ // next GC cycle.
+ _heap->free_set()->move_regions_from_collector_to_mutator(cset_regions);
+ }
+ // If !CONCURRENT, there's no value in expanding Mutator free set
+
+ ShenandoahHeapRegion* r = _regions->next();
+ // We update references for global, old, and young collections.
+ ShenandoahGeneration* const gc_generation = _heap->gc_generation();
+ shenandoah_assert_generations_reconciled();
+ assert(gc_generation->is_mark_complete(), "Expected complete marking");
+ ShenandoahMarkingContext* const ctx = _heap->marking_context();
+ bool is_mixed = _heap->collection_set()->has_old_regions();
+ while (r != nullptr) {
+ HeapWord* update_watermark = r->get_update_watermark();
+ assert(update_watermark >= r->bottom(), "sanity");
+
+ log_debug(gc)("Update refs worker " UINT32_FORMAT ", looking at region " SIZE_FORMAT, worker_id, r->index());
+ bool region_progress = false;
+ if (r->is_active() && !r->is_cset()) {
+ if (r->is_young()) {
+ _heap->marked_object_oop_iterate(r, &cl, update_watermark);
+ region_progress = true;
+ } else if (r->is_old()) {
+ if (gc_generation->is_global()) {
+
+ _heap->marked_object_oop_iterate(r, &cl, update_watermark);
+ region_progress = true;
+ }
+ // Otherwise, this is an old region in a young or mixed cycle. Process it during a second phase, below.
+ // Don't bother to report pacing progress in this case.
+ } else {
+ // Because updating of references runs concurrently, it is possible that a FREE inactive region transitions
+ // to a non-free active region while this loop is executing. Whenever this happens, the changing of a region's
+ // active status may propagate at a different speed than the changing of the region's affiliation.
+
+ // When we reach this control point, it is because a race has allowed a region's is_active() status to be seen
+ // by this thread before the region's affiliation() is seen by this thread.
+
+ // It's ok for this race to occur because the newly transformed region does not have any references to be
+ // updated.
+
+ assert(r->get_update_watermark() == r->bottom(),
+ "%s Region " SIZE_FORMAT " is_active but not recognized as YOUNG or OLD so must be newly transitioned from FREE",
+ r->affiliation_name(), r->index());
+ }
+ }
+
+ if (region_progress && ShenandoahPacing) {
+ _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
+ }
+
+ if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) {
+ return;
+ }
+
+ r = _regions->next();
+ }
+
+ if (!gc_generation->is_global()) {
+ // Since this is generational and not GLOBAL, we have to process the remembered set. There's no remembered
+ // set processing if not in generational mode or if GLOBAL mode.
+
+ // After this thread has exhausted its traditional update-refs work, it continues with updating refs within
+ // remembered set. The remembered set workload is better balanced between threads, so threads that are "behind"
+ // can catch up with other threads during this phase, allowing all threads to work more effectively in parallel.
+ update_references_in_remembered_set(worker_id, cl, ctx, is_mixed);
+ }
+ }
+
+ template
+ void update_references_in_remembered_set(uint worker_id, T &cl, const ShenandoahMarkingContext* ctx, bool is_mixed) {
+
+ struct ShenandoahRegionChunk assignment;
+ ShenandoahScanRemembered* scanner = _heap->old_generation()->card_scan();
+
+ while (!_heap->check_cancelled_gc_and_yield(CONCURRENT) && _work_chunks->next(&assignment)) {
+ // Keep grabbing next work chunk to process until finished, or asked to yield
+ ShenandoahHeapRegion* r = assignment._r;
+ if (r->is_active() && !r->is_cset() && r->is_old()) {
+ HeapWord* start_of_range = r->bottom() + assignment._chunk_offset;
+ HeapWord* end_of_range = r->get_update_watermark();
+ if (end_of_range > start_of_range + assignment._chunk_size) {
+ end_of_range = start_of_range + assignment._chunk_size;
+ }
+
+ if (start_of_range >= end_of_range) {
+ continue;
+ }
+
+ // Old region in a young cycle or mixed cycle.
+ if (is_mixed) {
+ if (r->is_humongous()) {
+ // Need to examine both dirty and clean cards during mixed evac.
+ r->oop_iterate_humongous_slice_all(&cl,start_of_range, assignment._chunk_size);
+ } else {
+ // Since this is mixed evacuation, old regions that are candidates for collection have not been coalesced
+ // and filled. This will use mark bits to find objects that need to be updated.
+ update_references_in_old_region(cl, ctx, scanner, r, start_of_range, end_of_range);
+ }
+ } else {
+ // This is a young evacuation
+ size_t cluster_size = CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster;
+ size_t clusters = assignment._chunk_size / cluster_size;
+ assert(clusters * cluster_size == assignment._chunk_size, "Chunk assignment must align on cluster boundaries");
+ scanner->process_region_slice(r, assignment._chunk_offset, clusters, end_of_range, &cl, true, worker_id);
+ }
+
+ if (ShenandoahPacing) {
+ _heap->pacer()->report_updaterefs(pointer_delta(end_of_range, start_of_range));
+ }
+ }
+ }
+ }
+
+ template
+ void update_references_in_old_region(T &cl, const ShenandoahMarkingContext* ctx, ShenandoahScanRemembered* scanner,
+ const ShenandoahHeapRegion* r, HeapWord* start_of_range,
+ HeapWord* end_of_range) const {
+ // In case last object in my range spans boundary of my chunk, I may need to scan all the way to top()
+ ShenandoahObjectToOopBoundedClosure objs(&cl, start_of_range, r->top());
+
+ // Any object that begins in a previous range is part of a different scanning assignment. Any object that
+ // starts after end_of_range is also not my responsibility. (Either allocated during evacuation, so does
+ // not hold pointers to from-space, or is beyond the range of my assigned work chunk.)
+
+ // Find the first object that begins in my range, if there is one. Note that `p` will be set to `end_of_range`
+ // when no live object is found in the range.
+ HeapWord* tams = ctx->top_at_mark_start(r);
+ HeapWord* p = get_first_object_start_word(ctx, scanner, tams, start_of_range, end_of_range);
+
+ while (p < end_of_range) {
+ // p is known to point to the beginning of marked object obj
+ oop obj = cast_to_oop(p);
+ objs.do_object(obj);
+ HeapWord* prev_p = p;
+ p += obj->size();
+ if (p < tams) {
+ p = ctx->get_next_marked_addr(p, tams);
+ // If there are no more marked objects before tams, this returns tams. Note that tams is
+ // either >= end_of_range, or tams is the start of an object that is marked.
+ }
+ assert(p != prev_p, "Lack of forward progress");
+ }
+ }
+
+ HeapWord* get_first_object_start_word(const ShenandoahMarkingContext* ctx, ShenandoahScanRemembered* scanner, HeapWord* tams,
+ HeapWord* start_of_range, HeapWord* end_of_range) const {
+ HeapWord* p = start_of_range;
+
+ if (p >= tams) {
+ // We cannot use ctx->is_marked(obj) to test whether an object begins at this address. Instead,
+ // we need to use the remembered set crossing map to advance p to the first object that starts
+ // within the enclosing card.
+ size_t card_index = scanner->card_index_for_addr(start_of_range);
+ while (true) {
+ HeapWord* first_object = scanner->first_object_in_card(card_index);
+ if (first_object != nullptr) {
+ p = first_object;
+ break;
+ } else if (scanner->addr_for_card_index(card_index + 1) < end_of_range) {
+ card_index++;
+ } else {
+ // Signal that no object was found in range
+ p = end_of_range;
+ break;
+ }
+ }
+ } else if (!ctx->is_marked(cast_to_oop(p))) {
+ p = ctx->get_next_marked_addr(p, tams);
+ // If there are no more marked objects before tams, this returns tams.
+ // Note that tams is either >= end_of_range, or tams is the start of an object that is marked.
+ }
+ return p;
+ }
+};
+
+void ShenandoahGenerationalHeap::update_heap_references(bool concurrent) {
+ assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
+ const uint nworkers = workers()->active_workers();
+ ShenandoahRegionChunkIterator work_list(nworkers);
+ if (concurrent) {
+ ShenandoahGenerationalUpdateHeapRefsTask task(&_update_refs_iterator, &work_list);
+ workers()->run_task(&task);
+ } else {
+ ShenandoahGenerationalUpdateHeapRefsTask task(&_update_refs_iterator, &work_list);
+ workers()->run_task(&task);
+ }
+
+ if (ShenandoahEnableCardStats) {
+ // Only do this if we are collecting card stats
+ ShenandoahScanRemembered* card_scan = old_generation()->card_scan();
+ assert(card_scan != nullptr, "Card table must exist when card stats are enabled");
+ card_scan->log_card_stats(nworkers, CARD_STAT_UPDATE_REFS);
+ }
+}
+
+struct ShenandoahCompositeRegionClosure {
+ template
+ class Closure : public ShenandoahHeapRegionClosure {
+ private:
+ C1 &_c1;
+ C2 &_c2;
+
+ public:
+ Closure(C1 &c1, C2 &c2) : ShenandoahHeapRegionClosure(), _c1(c1), _c2(c2) {}
+
+ void heap_region_do(ShenandoahHeapRegion* r) override {
+ _c1.heap_region_do(r);
+ _c2.heap_region_do(r);
+ }
+
+ bool is_thread_safe() override {
+ return _c1.is_thread_safe() && _c2.is_thread_safe();
+ }
+ };
+
+ template
+ static Closure of(C1 &c1, C2 &c2) {
+ return Closure(c1, c2);
+ }
+};
+
+class ShenandoahUpdateRegionAges : public ShenandoahHeapRegionClosure {
+private:
+ ShenandoahMarkingContext* _ctx;
+
+public:
+ explicit ShenandoahUpdateRegionAges(ShenandoahMarkingContext* ctx) : _ctx(ctx) { }
+
+ void heap_region_do(ShenandoahHeapRegion* r) override {
+ // Maintenance of region age must follow evacuation in order to account for
+ // evacuation allocations within survivor regions. We consult region age during
+ // the subsequent evacuation to determine whether certain objects need to
+ // be promoted.
+ if (r->is_young() && r->is_active()) {
+ HeapWord *tams = _ctx->top_at_mark_start(r);
+ HeapWord *top = r->top();
+
+ // Allocations move the watermark when top moves. However, compacting
+ // objects will sometimes lower top beneath the watermark, after which,
+ // attempts to read the watermark will assert out (watermark should not be
+ // higher than top).
+ if (top > tams) {
+ // There have been allocations in this region since the start of the cycle.
+ // Any objects new to this region must not assimilate elevated age.
+ r->reset_age();
+ } else if (ShenandoahGenerationalHeap::heap()->is_aging_cycle()) {
+ r->increment_age();
+ }
+ }
+ }
+
+ bool is_thread_safe() override {
+ return true;
+ }
+};
+
+void ShenandoahGenerationalHeap::final_update_refs_update_region_states() {
+ ShenandoahSynchronizePinnedRegionStates pins;
+ ShenandoahUpdateRegionAges ages(active_generation()->complete_marking_context());
+ auto cl = ShenandoahCompositeRegionClosure::of(pins, ages);
+ parallel_heap_region_iterate(&cl);
+}
+
+void ShenandoahGenerationalHeap::complete_degenerated_cycle() {
+ shenandoah_assert_heaplocked_or_safepoint();
+ if (is_concurrent_old_mark_in_progress()) {
+ // This is still necessary for degenerated cycles because the degeneration point may occur
+ // after final mark of the young generation. See ShenandoahConcurrentGC::op_final_updaterefs for
+ // a more detailed explanation.
+ old_generation()->transfer_pointers_from_satb();
+ }
+
+ // We defer generation resizing actions until after cset regions have been recycled.
+ TransferResult result = balance_generations();
+ LogTarget(Info, gc, ergo) lt;
+ if (lt.is_enabled()) {
+ LogStream ls(lt);
+ result.print_on("Degenerated GC", &ls);
+ }
+
+ // In case degeneration interrupted concurrent evacuation or update references, we need to clean up
+ // transient state. Otherwise, these actions have no effect.
+ reset_generation_reserves();
+
+ if (!old_generation()->is_parsable()) {
+ ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_coalesce_and_fill);
+ coalesce_and_fill_old_regions(false);
+ }
+}
+
+void ShenandoahGenerationalHeap::complete_concurrent_cycle() {
+ if (!old_generation()->is_parsable()) {
+ // Class unloading may render the card offsets unusable, so we must rebuild them before
+ // the next remembered set scan. We _could_ let the control thread do this sometime after
+ // the global cycle has completed and before the next young collection, but under memory
+ // pressure the control thread may not have the time (that is, because it's running back
+ // to back GCs). In that scenario, we would have to make the old regions parsable before
+ // we could start a young collection. This could delay the start of the young cycle and
+ // throw off the heuristics.
+ entry_global_coalesce_and_fill();
+ }
+
+ TransferResult result;
+ {
+ ShenandoahHeapLocker locker(lock());
+
+ result = balance_generations();
+ reset_generation_reserves();
+ }
+
+ LogTarget(Info, gc, ergo) lt;
+ if (lt.is_enabled()) {
+ LogStream ls(lt);
+ result.print_on("Concurrent GC", &ls);
+ }
+}
+
+void ShenandoahGenerationalHeap::entry_global_coalesce_and_fill() {
+ const char* msg = "Coalescing and filling old regions";
+ ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_coalesce_and_fill);
+
+ TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
+ EventMark em("%s", msg);
+ ShenandoahWorkerScope scope(workers(),
+ ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
+ "concurrent coalesce and fill");
+
+ coalesce_and_fill_old_regions(true);
+}
+
+void ShenandoahGenerationalHeap::update_region_ages(ShenandoahMarkingContext* ctx) {
+ ShenandoahUpdateRegionAges cl(ctx);
+ parallel_heap_region_iterate(&cl);
+}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp
new file mode 100644
index 00000000000..cef5dfd7070
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp
@@ -0,0 +1,169 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONALHEAP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONALHEAP
+
+#include "gc/shenandoah/shenandoahAsserts.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "memory/universe.hpp"
+#include "utilities/checkedCast.hpp"
+
+class PLAB;
+class ShenandoahRegulatorThread;
+class ShenandoahGenerationalControlThread;
+class ShenandoahAgeCensus;
+
+class ShenandoahGenerationalHeap : public ShenandoahHeap {
+public:
+ explicit ShenandoahGenerationalHeap(ShenandoahCollectorPolicy* policy);
+ void post_initialize() override;
+ void initialize_heuristics() override;
+
+ static ShenandoahGenerationalHeap* heap() {
+ shenandoah_assert_generational();
+ CollectedHeap* heap = Universe::heap();
+ return cast(heap);
+ }
+
+ static ShenandoahGenerationalHeap* cast(CollectedHeap* heap) {
+ shenandoah_assert_generational();
+ return checked_cast(heap);
+ }
+
+ void print_init_logger() const override;
+ void print_tracing_info() const override;
+
+ size_t unsafe_max_tlab_alloc(Thread *thread) const override;
+
+private:
+ // ---------- Evacuations and Promotions
+ //
+ // True when regions and objects should be aged during the current cycle
+ ShenandoahSharedFlag _is_aging_cycle;
+ // Age census used for adapting tenuring threshold
+ ShenandoahAgeCensus* _age_census;
+ // Used primarily to look for failed evacuation attempts.
+ ShenandoahEvacuationTracker* _evac_tracker;
+
+public:
+ void set_aging_cycle(bool cond) {
+ _is_aging_cycle.set_cond(cond);
+ }
+
+ inline bool is_aging_cycle() const {
+ return _is_aging_cycle.is_set();
+ }
+
+ // Return the age census object for young gen
+ ShenandoahAgeCensus* age_census() const {
+ return _age_census;
+ }
+
+ ShenandoahEvacuationTracker* evac_tracker() const {
+ return _evac_tracker;
+ }
+
+ // Ages regions that haven't been used for allocations in the current cycle.
+ // Resets ages for regions that have been used for allocations.
+ void update_region_ages(ShenandoahMarkingContext* ctx);
+
+ oop evacuate_object(oop p, Thread* thread) override;
+ oop try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahAffiliation target_gen);
+ void evacuate_collection_set(bool concurrent) override;
+ void promote_regions_in_place(bool concurrent);
+
+ size_t plab_min_size() const { return _min_plab_size; }
+ size_t plab_max_size() const { return _max_plab_size; }
+
+ void retire_plab(PLAB* plab);
+ void retire_plab(PLAB* plab, Thread* thread);
+
+ // ---------- Update References
+ //
+ void update_heap_references(bool concurrent) override;
+ void final_update_refs_update_region_states() override;
+
+private:
+ HeapWord* allocate_from_plab(Thread* thread, size_t size, bool is_promotion);
+ HeapWord* allocate_from_plab_slow(Thread* thread, size_t size, bool is_promotion);
+ HeapWord* allocate_new_plab(size_t min_size, size_t word_size, size_t* actual_size);
+
+ const size_t _min_plab_size;
+ const size_t _max_plab_size;
+
+ static size_t calculate_min_plab();
+ static size_t calculate_max_plab();
+
+public:
+ // ---------- Serviceability
+ //
+ void initialize_serviceability() override;
+ GrowableArray memory_pools() override;
+
+ ShenandoahRegulatorThread* regulator_thread() const { return _regulator_thread; }
+
+ void gc_threads_do(ThreadClosure* tcl) const override;
+
+ void stop() override;
+
+ // Used for logging the result of a region transfer outside the heap lock
+ struct TransferResult {
+ bool success;
+ size_t region_count;
+ const char* region_destination;
+
+ void print_on(const char* when, outputStream* ss) const;
+ };
+
+ const ShenandoahGenerationSizer* generation_sizer() const { return &_generation_sizer; }
+
+ // Zeros out the evacuation and promotion reserves
+ void reset_generation_reserves();
+
+ // Computes the optimal size for the old generation, represented as a surplus or deficit of old regions
+ void compute_old_generation_balance(size_t old_xfer_limit, size_t old_cset_regions);
+
+ // Transfers surplus old regions to young, or takes regions from young to satisfy old region deficit
+ TransferResult balance_generations();
+
+ // Balances generations, coalesces and fills old regions if necessary
+ void complete_degenerated_cycle();
+ void complete_concurrent_cycle();
+private:
+ void initialize_controller() override;
+ void entry_global_coalesce_and_fill();
+
+ // Makes old regions parsable. This will also rebuild card offsets, which is necessary if classes were unloaded
+ void coalesce_and_fill_old_regions(bool concurrent);
+
+ ShenandoahRegulatorThread* _regulator_thread;
+
+ MemoryPool* _young_gen_memory_pool;
+ MemoryPool* _old_gen_memory_pool;
+
+ ShenandoahGenerationSizer _generation_sizer;
+};
+
+#endif //SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONALHEAP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp
new file mode 100644
index 00000000000..9b13c7c95af
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp
@@ -0,0 +1,146 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+#include "precompiled.hpp"
+
+#include "gc/shenandoah/shenandoahAgeCensus.hpp"
+#include "gc/shenandoah/heuristics/shenandoahGlobalHeuristics.hpp"
+#include "gc/shenandoah/shenandoahFreeSet.hpp"
+#include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
+#include "gc/shenandoah/shenandoahUtils.hpp"
+#include "gc/shenandoah/shenandoahVerifier.hpp"
+
+
+const char* ShenandoahGlobalGeneration::name() const {
+ return type() == NON_GEN ? "" : "Global";
+}
+
+size_t ShenandoahGlobalGeneration::max_capacity() const {
+ return ShenandoahHeap::heap()->max_capacity();
+}
+
+size_t ShenandoahGlobalGeneration::used_regions() const {
+ ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
+ assert(heap->mode()->is_generational(), "Region usage accounting is only for generational mode");
+ return heap->old_generation()->used_regions() + heap->young_generation()->used_regions();
+}
+
+size_t ShenandoahGlobalGeneration::used_regions_size() const {
+ return ShenandoahHeap::heap()->capacity();
+}
+
+size_t ShenandoahGlobalGeneration::soft_max_capacity() const {
+ return ShenandoahHeap::heap()->soft_max_capacity();
+}
+
+size_t ShenandoahGlobalGeneration::available() const {
+ return ShenandoahHeap::heap()->free_set()->available();
+}
+
+size_t ShenandoahGlobalGeneration::soft_available() const {
+ size_t available = this->available();
+
+ // Make sure the code below treats available without the soft tail.
+ assert(max_capacity() >= soft_max_capacity(), "Max capacity must be greater than soft max capacity.");
+ size_t soft_tail = max_capacity() - soft_max_capacity();
+ return (available > soft_tail) ? (available - soft_tail) : 0;
+}
+
+void ShenandoahGlobalGeneration::set_concurrent_mark_in_progress(bool in_progress) {
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+ if (in_progress && heap->mode()->is_generational()) {
+ // Global collection has preempted an old generation mark. This is fine
+ // because the global generation includes the old generation, but we
+ // want the global collect to start from a clean slate and we don't want
+ // any stale state in the old generation.
+ assert(!heap->is_concurrent_old_mark_in_progress(), "Old cycle should not be running.");
+ }
+
+ heap->set_concurrent_young_mark_in_progress(in_progress);
+}
+
+bool ShenandoahGlobalGeneration::contains(ShenandoahAffiliation affiliation) const {
+ return true;
+}
+
+bool ShenandoahGlobalGeneration::contains(ShenandoahHeapRegion* region) const {
+ return true;
+}
+
+void ShenandoahGlobalGeneration::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) {
+ ShenandoahHeap::heap()->parallel_heap_region_iterate(cl);
+}
+
+void ShenandoahGlobalGeneration::heap_region_iterate(ShenandoahHeapRegionClosure* cl) {
+ ShenandoahHeap::heap()->heap_region_iterate(cl);
+}
+
+bool ShenandoahGlobalGeneration::is_concurrent_mark_in_progress() {
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+ return heap->is_concurrent_mark_in_progress();
+}
+
+ShenandoahHeuristics* ShenandoahGlobalGeneration::initialize_heuristics(ShenandoahMode* gc_mode) {
+ if (gc_mode->is_generational()) {
+ _heuristics = new ShenandoahGlobalHeuristics(this);
+ } else {
+ _heuristics = gc_mode->initialize_heuristics(this);
+ }
+
+ _heuristics->set_guaranteed_gc_interval(ShenandoahGuaranteedGCInterval);
+ confirm_heuristics_mode();
+ return _heuristics;
+}
+
+void ShenandoahGlobalGeneration::set_mark_complete() {
+ ShenandoahGeneration::set_mark_complete();
+ if (ShenandoahHeap::heap()->mode()->is_generational()) {
+ ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
+ heap->young_generation()->set_mark_complete();
+ heap->old_generation()->set_mark_complete();
+ }
+}
+
+void ShenandoahGlobalGeneration::set_mark_incomplete() {
+ ShenandoahGeneration::set_mark_incomplete();
+ if (ShenandoahHeap::heap()->mode()->is_generational()) {
+ ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
+ heap->young_generation()->set_mark_incomplete();
+ heap->old_generation()->set_mark_incomplete();
+ }
+}
+
+void ShenandoahGlobalGeneration::prepare_gc() {
+ ShenandoahGeneration::prepare_gc();
+
+ if (ShenandoahHeap::heap()->mode()->is_generational()) {
+ assert(type() == GLOBAL, "Unexpected generation type");
+ // Clear any stale/partial local census data before the start of a
+ // new marking cycle
+ ShenandoahGenerationalHeap::heap()->age_census()->reset_local();
+ } else {
+ assert(type() == NON_GEN, "Unexpected generation type");
+ }
+}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp
new file mode 100644
index 00000000000..d51a77fdf8f
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp
@@ -0,0 +1,73 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHGLOBALGENERATION_HPP
+#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHGLOBALGENERATION_HPP
+
+#include "gc/shenandoah/shenandoahGeneration.hpp"
+#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
+
+// A "generation" that represents the whole heap.
+class ShenandoahGlobalGeneration : public ShenandoahGeneration {
+public:
+ ShenandoahGlobalGeneration(bool generational, uint max_queues, size_t max_capacity, size_t soft_max_capacity)
+ : ShenandoahGeneration(generational ? GLOBAL : NON_GEN, max_queues, max_capacity, soft_max_capacity) { }
+
+public:
+ const char* name() const override;
+
+ size_t max_capacity() const override;
+ size_t soft_max_capacity() const override;
+ size_t used_regions() const override;
+ size_t used_regions_size() const override;
+ size_t available() const override;
+ size_t soft_available() const override;
+
+ void set_concurrent_mark_in_progress(bool in_progress) override;
+
+ bool contains(ShenandoahAffiliation affiliation) const override;
+ bool contains(ShenandoahHeapRegion* region) const override;
+
+ bool contains(oop obj) const override {
+ return ShenandoahHeap::heap()->is_in_reserved(obj);
+ }
+
+ void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) override;
+
+ void heap_region_iterate(ShenandoahHeapRegionClosure* cl) override;
+
+ bool is_concurrent_mark_in_progress() override;
+
+ void set_mark_complete() override;
+
+ void set_mark_incomplete() override;
+
+ ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode) override;
+
+ virtual void prepare_gc() override;
+};
+
+#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHGLOBALGENERATION_HPP
+
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
index 0f7139691a3..6ef66926b72 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -37,6 +38,9 @@
#include "gc/shared/plab.hpp"
#include "gc/shared/tlab_globals.hpp"
+#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp"
+#include "gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp"
+#include "gc/shenandoah/shenandoahAllocRequest.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.hpp"
@@ -45,20 +49,25 @@
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
#include "gc/shenandoah/shenandoahControlThread.hpp"
#include "gc/shenandoah/shenandoahFreeSet.hpp"
+#include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
+#include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
#include "gc/shenandoah/shenandoahInitLogger.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
#include "gc/shenandoah/shenandoahMemoryPool.hpp"
-#include "gc/shenandoah/shenandoahMetrics.hpp"
#include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
#include "gc/shenandoah/shenandoahPacer.inline.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
#include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
#include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
#include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
+#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
#include "gc/shenandoah/shenandoahSTWMark.hpp"
#include "gc/shenandoah/shenandoahUtils.hpp"
#include "gc/shenandoah/shenandoahVerifier.hpp"
@@ -66,8 +75,12 @@
#include "gc/shenandoah/shenandoahVMOperations.hpp"
#include "gc/shenandoah/shenandoahWorkGroup.hpp"
#include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
+#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
+#include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp"
#include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
#include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
+#include "utilities/globalDefinitions.hpp"
+
#if INCLUDE_JFR
#include "gc/shenandoah/shenandoahJfrSupport.hpp"
#endif
@@ -161,9 +174,6 @@ jint ShenandoahHeap::initialize() {
"Regions should cover entire heap exactly: " SIZE_FORMAT " != " SIZE_FORMAT "/" SIZE_FORMAT,
_num_regions, max_byte_size, reg_size_bytes);
- // Now we know the number of regions, initialize the heuristics.
- initialize_heuristics();
-
size_t num_committed_regions = init_byte_size / reg_size_bytes;
num_committed_regions = MIN2(num_committed_regions, _num_regions);
assert(num_committed_regions <= _num_regions, "sanity");
@@ -217,6 +227,28 @@ jint ShenandoahHeap::initialize() {
"Cannot commit heap memory");
}
+ BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this, _heap_region));
+
+ // Now we know the number of regions and heap sizes, initialize the heuristics.
+ initialize_heuristics();
+
+ assert(_heap_region.byte_size() == heap_rs.size(), "Need to know reserved size for card table");
+
+ //
+ // Worker threads must be initialized after the barrier is configured
+ //
+ _workers = new ShenandoahWorkerThreads("Shenandoah GC Threads", _max_workers);
+ if (_workers == nullptr) {
+ vm_exit_during_initialization("Failed necessary allocation.");
+ } else {
+ _workers->initialize_workers();
+ }
+
+ if (ParallelGCThreads > 1) {
+ _safepoint_workers = new ShenandoahWorkerThreads("Safepoint Cleanup Thread", ParallelGCThreads);
+ _safepoint_workers->initialize_workers();
+ }
+
//
// Reserve and commit memory for bitmap(s)
//
@@ -257,14 +289,14 @@ jint ShenandoahHeap::initialize() {
_bitmap_region_special = bitmap.special();
size_t bitmap_init_commit = _bitmap_bytes_per_slice *
- align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
+ align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
if (!_bitmap_region_special) {
os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
"Cannot commit bitmap memory");
}
- _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions, _max_workers);
+ _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
if (ShenandoahVerify) {
ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
@@ -348,6 +380,7 @@ jint ShenandoahHeap::initialize() {
}
_regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
+ _affiliations = NEW_C_HEAP_ARRAY(uint8_t, _num_regions, mtGC);
_free_set = new ShenandoahFreeSet(this, _num_regions);
{
@@ -364,12 +397,18 @@ jint ShenandoahHeap::initialize() {
_marking_context->initialize_top_at_mark_start(r);
_regions[i] = r;
assert(!collection_set()->is_in(i), "New region should not be in collection set");
+
+ _affiliations[i] = ShenandoahAffiliation::FREE;
}
// Initialize to complete
_marking_context->mark_complete();
+ size_t young_cset_regions, old_cset_regions;
- _free_set->rebuild();
+ // We are initializing free set. We ignore cset region tallies.
+ size_t first_old, last_old, num_old;
+ _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old);
+ _free_set->finish_rebuild(young_cset_regions, old_cset_regions, num_old);
}
if (AlwaysPreTouch) {
@@ -418,21 +457,31 @@ jint ShenandoahHeap::initialize() {
_pacer->setup_for_idle();
}
- _control_thread = new ShenandoahControlThread();
+ initialize_controller();
- ShenandoahInitLogger::print();
+ print_init_logger();
FullGCForwarding::initialize(_heap_region);
return JNI_OK;
}
+void ShenandoahHeap::initialize_controller() {
+ _control_thread = new ShenandoahControlThread();
+}
+
+void ShenandoahHeap::print_init_logger() const {
+ ShenandoahInitLogger::print();
+}
+
void ShenandoahHeap::initialize_mode() {
if (ShenandoahGCMode != nullptr) {
if (strcmp(ShenandoahGCMode, "satb") == 0) {
_gc_mode = new ShenandoahSATBMode();
} else if (strcmp(ShenandoahGCMode, "passive") == 0) {
_gc_mode = new ShenandoahPassiveMode();
+ } else if (strcmp(ShenandoahGCMode, "generational") == 0) {
+ _gc_mode = new ShenandoahGenerationalMode();
} else {
vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
}
@@ -453,19 +502,8 @@ void ShenandoahHeap::initialize_mode() {
}
void ShenandoahHeap::initialize_heuristics() {
- assert(_gc_mode != nullptr, "Must be initialized");
- _heuristics = _gc_mode->initialize_heuristics();
-
- if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
- vm_exit_during_initialization(
- err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
- _heuristics->name()));
- }
- if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
- vm_exit_during_initialization(
- err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
- _heuristics->name()));
- }
+ _global_generation = new ShenandoahGlobalGeneration(mode()->is_generational(), max_workers(), max_capacity(), max_capacity());
+ _global_generation->initialize_heuristics(mode());
}
#ifdef _MSC_VER
@@ -475,34 +513,38 @@ void ShenandoahHeap::initialize_heuristics() {
ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
CollectedHeap(),
+ _gc_generation(nullptr),
+ _active_generation(nullptr),
_initial_size(0),
- _used(0),
_committed(0),
- _bytes_allocated_since_gc_start(0),
- _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
+ _max_workers(MAX3(ConcGCThreads, ParallelGCThreads, 1U)),
_workers(nullptr),
_safepoint_workers(nullptr),
_heap_region_special(false),
_num_regions(0),
_regions(nullptr),
- _update_refs_iterator(this),
+ _affiliations(nullptr),
_gc_state_changed(false),
_gc_no_progress_count(0),
+ _cancel_requested_time(0),
+ _update_refs_iterator(this),
+ _global_generation(nullptr),
_control_thread(nullptr),
+ _young_generation(nullptr),
+ _old_generation(nullptr),
_shenandoah_policy(policy),
_gc_mode(nullptr),
- _heuristics(nullptr),
_free_set(nullptr),
_pacer(nullptr),
_verifier(nullptr),
_phase_timings(nullptr),
+ _mmu_tracker(),
_monitoring_support(nullptr),
_memory_pool(nullptr),
_stw_memory_manager("Shenandoah Pauses"),
_cycle_memory_manager("Shenandoah Cycles"),
_gc_timer(new ConcurrentGCTimer()),
_log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
- _ref_processor(new ShenandoahReferenceProcessor(MAX2(_max_workers, 1U))),
_marking_context(nullptr),
_bitmap_size(0),
_bitmap_regions_per_slice(0),
@@ -512,58 +554,14 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
_liveness_cache(nullptr),
_collection_set(nullptr)
{
- // Initialize GC mode early, so we can adjust barrier support
+ // Initialize GC mode early, many subsequent initialization procedures depend on it
initialize_mode();
- BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
-
- _max_workers = MAX2(_max_workers, 1U);
- _workers = new ShenandoahWorkerThreads("Shenandoah GC Threads", _max_workers);
- if (_workers == nullptr) {
- vm_exit_during_initialization("Failed necessary allocation.");
- } else {
- _workers->initialize_workers();
- }
-
- if (ParallelGCThreads > 1) {
- _safepoint_workers = new ShenandoahWorkerThreads("Safepoint Cleanup Thread",
- ParallelGCThreads);
- _safepoint_workers->initialize_workers();
- }
}
#ifdef _MSC_VER
#pragma warning( pop )
#endif
-class ShenandoahResetBitmapTask : public WorkerTask {
-private:
- ShenandoahRegionIterator _regions;
-
-public:
- ShenandoahResetBitmapTask() :
- WorkerTask("Shenandoah Reset Bitmap") {}
-
- void work(uint worker_id) {
- ShenandoahHeapRegion* region = _regions.next();
- ShenandoahHeap* heap = ShenandoahHeap::heap();
- ShenandoahMarkingContext* const ctx = heap->marking_context();
- while (region != nullptr) {
- if (heap->is_bitmap_slice_committed(region)) {
- ctx->clear_bitmap(region);
- }
- region = _regions.next();
- }
- }
-};
-
-void ShenandoahHeap::reset_mark_bitmap() {
- assert_gc_workers(_workers->active_workers());
- mark_incomplete_marking_context();
-
- ShenandoahResetBitmapTask task;
- _workers->run_task(&task);
-}
-
void ShenandoahHeap::print_on(outputStream* st) const {
st->print_cr("Shenandoah Heap");
st->print_cr(" " SIZE_FORMAT "%s max, " SIZE_FORMAT "%s soft max, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
@@ -578,7 +576,12 @@ void ShenandoahHeap::print_on(outputStream* st) const {
st->print("Status: ");
if (has_forwarded_objects()) st->print("has forwarded objects, ");
- if (is_concurrent_mark_in_progress()) st->print("marking, ");
+ if (!mode()->is_generational()) {
+ if (is_concurrent_mark_in_progress()) st->print("marking,");
+ } else {
+ if (is_concurrent_old_mark_in_progress()) st->print("old marking, ");
+ if (is_concurrent_young_mark_in_progress()) st->print("young marking, ");
+ }
if (is_evacuation_in_progress()) st->print("evacuating, ");
if (is_update_refs_in_progress()) st->print("updating refs, ");
if (is_degenerated_gc_in_progress()) st->print("degenerated gc, ");
@@ -629,6 +632,8 @@ class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
void ShenandoahHeap::post_initialize() {
CollectedHeap::post_initialize();
+ _mmu_tracker.initialize();
+
MutexLocker ml(Threads_lock);
ShenandoahInitWorkerGCLABClosure init_gclabs;
@@ -642,23 +647,21 @@ void ShenandoahHeap::post_initialize() {
_safepoint_workers->set_initialize_gclab();
}
- _heuristics->initialize();
-
JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers();)
}
+ShenandoahHeuristics* ShenandoahHeap::heuristics() {
+ return _global_generation->heuristics();
+}
+
size_t ShenandoahHeap::used() const {
- return Atomic::load(&_used);
+ return global_generation()->used();
}
size_t ShenandoahHeap::committed() const {
return Atomic::load(&_committed);
}
-size_t ShenandoahHeap::available() const {
- return free_set()->available();
-}
-
void ShenandoahHeap::increase_committed(size_t bytes) {
shenandoah_assert_heaplocked_or_safepoint();
_committed += bytes;
@@ -669,33 +672,84 @@ void ShenandoahHeap::decrease_committed(size_t bytes) {
_committed -= bytes;
}
-void ShenandoahHeap::increase_used(size_t bytes) {
- Atomic::add(&_used, bytes, memory_order_relaxed);
+// For tracking usage based on allocations, it should be the case that:
+// * The sum of regions::used == heap::used
+// * The sum of a generation's regions::used == generation::used
+// * The sum of a generation's humongous regions::free == generation::humongous_waste
+// These invariants are checked by the verifier on GC safepoints.
+//
+// Additional notes:
+// * When a mutator's allocation request causes a region to be retired, the
+// free memory left in that region is considered waste. It does not contribute
+// to the usage, but it _does_ contribute to allocation rate.
+// * The bottom of a PLAB must be aligned on card size. In some cases this will
+// require padding in front of the PLAB (a filler object). Because this padding
+// is included in the region's used memory we include the padding in the usage
+// accounting as waste.
+// * Mutator allocations are used to compute an allocation rate. They are also
+// sent to the Pacer for those purposes.
+// * There are three sources of waste:
+// 1. The padding used to align a PLAB on card size
+// 2. Region's free is less than minimum TLAB size and is retired
+// 3. The unused portion of memory in the last region of a humongous object
+void ShenandoahHeap::increase_used(const ShenandoahAllocRequest& req) {
+ size_t actual_bytes = req.actual_size() * HeapWordSize;
+ size_t wasted_bytes = req.waste() * HeapWordSize;
+ ShenandoahGeneration* generation = generation_for(req.affiliation());
+
+ if (req.is_gc_alloc()) {
+ assert(wasted_bytes == 0 || req.type() == ShenandoahAllocRequest::_alloc_plab, "Only PLABs have waste");
+ increase_used(generation, actual_bytes + wasted_bytes);
+ } else {
+ assert(req.is_mutator_alloc(), "Expected mutator alloc here");
+ // padding and actual size both count towards allocation counter
+ generation->increase_allocated(actual_bytes + wasted_bytes);
+
+ // only actual size counts toward usage for mutator allocations
+ increase_used(generation, actual_bytes);
+
+ // notify pacer of both actual size and waste
+ notify_mutator_alloc_words(req.actual_size(), req.waste());
+
+ if (wasted_bytes > 0 && ShenandoahHeapRegion::requires_humongous(req.actual_size())) {
+ increase_humongous_waste(generation,wasted_bytes);
+ }
+ }
}
-void ShenandoahHeap::set_used(size_t bytes) {
- Atomic::store(&_used, bytes);
+void ShenandoahHeap::increase_humongous_waste(ShenandoahGeneration* generation, size_t bytes) {
+ generation->increase_humongous_waste(bytes);
+ if (!generation->is_global()) {
+ global_generation()->increase_humongous_waste(bytes);
+ }
}
-void ShenandoahHeap::decrease_used(size_t bytes) {
- assert(used() >= bytes, "never decrease heap size by more than we've left");
- Atomic::sub(&_used, bytes, memory_order_relaxed);
+void ShenandoahHeap::decrease_humongous_waste(ShenandoahGeneration* generation, size_t bytes) {
+ generation->decrease_humongous_waste(bytes);
+ if (!generation->is_global()) {
+ global_generation()->decrease_humongous_waste(bytes);
+ }
}
-void ShenandoahHeap::increase_allocated(size_t bytes) {
- Atomic::add(&_bytes_allocated_since_gc_start, bytes, memory_order_relaxed);
+void ShenandoahHeap::increase_used(ShenandoahGeneration* generation, size_t bytes) {
+ generation->increase_used(bytes);
+ if (!generation->is_global()) {
+ global_generation()->increase_used(bytes);
+ }
}
-void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
- size_t bytes = words * HeapWordSize;
- if (!waste) {
- increase_used(bytes);
+void ShenandoahHeap::decrease_used(ShenandoahGeneration* generation, size_t bytes) {
+ generation->decrease_used(bytes);
+ if (!generation->is_global()) {
+ global_generation()->decrease_used(bytes);
}
- increase_allocated(bytes);
+}
+
+void ShenandoahHeap::notify_mutator_alloc_words(size_t words, size_t waste) {
if (ShenandoahPacing) {
control_thread()->pacing_notify_alloc(words);
- if (waste) {
- pacer()->claim_for_alloc(words);
+ if (waste > 0) {
+ pacer()->claim_for_alloc(waste);
}
}
}
@@ -825,8 +879,6 @@ void ShenandoahHeap::notify_heap_changed() {
// Update monitoring counters when we took a new region. This amortizes the
// update costs on slow path.
monitoring_support()->notify_heap_changed();
-
- // This is called from allocation path, and thus should be fast.
_heap_changed.try_set();
}
@@ -844,17 +896,20 @@ HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size)
// Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
+
new_size = MIN2(new_size, PLAB::max_size());
new_size = MAX2(new_size, PLAB::min_size());
// Record new heuristic value even if we take any shortcut. This captures
// the case when moderately-sized objects always take a shortcut. At some point,
// heuristics should catch up with them.
+ log_debug(gc, free)("Set new GCLAB size: " SIZE_FORMAT, new_size);
ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
if (new_size < size) {
// New size still does not fit the object. Fall back to shared allocation.
// This avoids retiring perfectly good GCLABs, when we encounter a large object.
+ log_debug(gc, free)("New gclab size (" SIZE_FORMAT ") is too small for " SIZE_FORMAT, new_size, size);
return nullptr;
}
@@ -884,6 +939,7 @@ HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size)
return gclab->allocate(size);
}
+// Called from stubs in JIT code or interpreter
HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
size_t requested_size,
size_t* actual_size) {
@@ -932,8 +988,10 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
// is testing that the GC overhead limit has not been exceeded.
// This will notify the collector to start a cycle, but will raise
// an OOME to the mutator if the last Full GCs have not made progress.
+ // gc_no_progress_count is incremented following each degen or full GC that fails to achieve is_good_progress().
if (result == nullptr && !req.is_lab_alloc() && get_gc_no_progress_count() > ShenandoahNoProgressThreshold) {
control_thread()->handle_alloc_failure(req, false);
+ req.set_actual_size(0);
return nullptr;
}
@@ -948,8 +1006,6 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
// Stop retrying and return nullptr to cause OOMError exception if our allocation failed even after:
// a) We experienced a GC that had good progress, or
// b) We experienced at least one Full GC (whether or not it had good progress)
- //
- // TODO: Consider GLOBAL GC rather than Full GC to remediate OOM condition: https://bugs.openjdk.org/browse/JDK-8335910
size_t original_count = shenandoah_policy()->full_gc_count();
while ((result == nullptr) && (original_count == shenandoah_policy()->full_gc_count())) {
@@ -960,7 +1016,7 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
// If our allocation request has been satisifed after it initially failed, we count this as good gc progress
notify_gc_progress();
}
- if (log_is_enabled(Debug, gc, alloc)) {
+ if (log_develop_is_enabled(Debug, gc, alloc)) {
ResourceMark rm;
log_debug(gc, alloc)("Thread: %s, Result: " PTR_FORMAT ", Request: %s, Size: " SIZE_FORMAT
", Original: " SIZE_FORMAT ", Latest: " SIZE_FORMAT,
@@ -979,6 +1035,14 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
notify_heap_changed();
}
+ if (result == nullptr) {
+ req.set_actual_size(0);
+ }
+
+ // This is called regardless of the outcome of the allocation to account
+ // for any waste created by retiring regions with this request.
+ increase_used(req);
+
if (result != nullptr) {
size_t requested = req.size();
size_t actual = req.actual_size();
@@ -988,16 +1052,12 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
if (req.is_mutator_alloc()) {
- notify_mutator_alloc_words(actual, false);
-
// If we requested more than we were granted, give the rest back to pacer.
// This only matters if we are in the same pacing epoch: do not try to unpace
// over the budget for the other phase.
if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
}
- } else {
- increase_used(actual*HeapWordSize);
}
}
@@ -1010,7 +1070,43 @@ HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req
// we are already running at safepoint or from stack watermark machinery, and we cannot
// block again.
ShenandoahHeapLocker locker(lock(), req.is_mutator_alloc());
- return _free_set->allocate(req, in_new_region);
+
+ // Make sure the old generation has room for either evacuations or promotions before trying to allocate.
+ if (req.is_old() && !old_generation()->can_allocate(req)) {
+ return nullptr;
+ }
+
+ // If TLAB request size is greater than available, allocate() will attempt to downsize request to fit within available
+ // memory.
+ HeapWord* result = _free_set->allocate(req, in_new_region);
+
+ // Record the plab configuration for this result and register the object.
+ if (result != nullptr && req.is_old()) {
+ old_generation()->configure_plab_for_current_thread(req);
+ if (req.type() == ShenandoahAllocRequest::_alloc_shared_gc) {
+ // Register the newly allocated object while we're holding the global lock since there's no synchronization
+ // built in to the implementation of register_object(). There are potential races when multiple independent
+ // threads are allocating objects, some of which might span the same card region. For example, consider
+ // a card table's memory region within which three objects are being allocated by three different threads:
+ //
+ // objects being "concurrently" allocated:
+ // [-----a------][-----b-----][--------------c------------------]
+ // [---- card table memory range --------------]
+ //
+ // Before any objects are allocated, this card's memory range holds no objects. Note that allocation of object a
+ // wants to set the starts-object, first-start, and last-start attributes of the preceding card region.
+ // Allocation of object b wants to set the starts-object, first-start, and last-start attributes of this card region.
+ // Allocation of object c also wants to set the starts-object, first-start, and last-start attributes of this
+ // card region.
+ //
+ // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as
+ // last-start representing object b while first-start represents object c. This is why we need to require all
+ // register_object() invocations to be "mutually exclusive" with respect to each card's memory range.
+ old_generation()->card_scan()->register_object(result);
+ }
+ }
+
+ return result;
}
HeapWord* ShenandoahHeap::mem_allocate(size_t size,
@@ -1025,8 +1121,8 @@ MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* lo
MetaWord* result;
// Inform metaspace OOM to GC heuristics if class unloading is possible.
- if (heuristics()->can_unload_classes()) {
- ShenandoahHeuristics* h = heuristics();
+ ShenandoahHeuristics* h = global_generation()->heuristics();
+ if (h->can_unload_classes()) {
h->record_metaspace_oom();
}
@@ -1124,20 +1220,29 @@ void ShenandoahHeap::evacuate_collection_set(bool concurrent) {
}
oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
- if (ShenandoahThreadLocalData::is_oom_during_evac(Thread::current())) {
- // This thread went through the OOM during evac protocol and it is safe to return
- // the forward pointer. It must not attempt to evacuate any more.
+ assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
+ if (ShenandoahThreadLocalData::is_oom_during_evac(thread)) {
+ // This thread went through the OOM during evac protocol. It is safe to return
+ // the forward pointer. It must not attempt to evacuate any other objects.
return ShenandoahBarrierSet::resolve_forwarded(p);
}
assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
- size_t size = ShenandoahForwarding::size(p);
+ ShenandoahHeapRegion* r = heap_region_containing(p);
+ assert(!r->is_humongous(), "never evacuate humongous objects");
- assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
+ ShenandoahAffiliation target_gen = r->affiliation();
+ return try_evacuate_object(p, thread, r, target_gen);
+}
- bool alloc_from_gclab = true;
+oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
+ ShenandoahAffiliation target_gen) {
+ assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
+ assert(from_region->is_young(), "Only expect evacuations from young in this mode");
+ bool alloc_from_lab = true;
HeapWord* copy = nullptr;
+ size_t size = ShenandoahForwarding::size(p);
#ifdef ASSERT
if (ShenandoahOOMDuringEvacALot &&
@@ -1149,9 +1254,10 @@ oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
copy = allocate_from_gclab(thread, size);
}
if (copy == nullptr) {
- ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size);
+ // If we failed to allocate in LAB, we'll try a shared allocation.
+ ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen);
copy = allocate_memory(req);
- alloc_from_gclab = false;
+ alloc_from_lab = false;
}
#ifdef ASSERT
}
@@ -1182,17 +1288,19 @@ oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
// But if it happens to contain references to evacuated regions, those references would
// not get updated for this stale copy during this cycle, and we will crash while scanning
// it the next cycle.
- //
- // For GCLAB allocations, it is enough to rollback the allocation ptr. Either the next
- // object will overwrite this stale copy, or the filler object on LAB retirement will
- // do this. For non-GCLAB allocations, we have no way to retract the allocation, and
- // have to explicitly overwrite the copy with the filler object. With that overwrite,
- // we have to keep the fwdptr initialized and pointing to our (stale) copy.
- if (alloc_from_gclab) {
+ if (alloc_from_lab) {
+ // For LAB allocations, it is enough to rollback the allocation ptr. Either the next
+ // object will overwrite this stale copy, or the filler object on LAB retirement will
+ // do this.
ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size);
} else {
+ // For non-LAB allocations, we have no way to retract the allocation, and
+ // have to explicitly overwrite the copy with the filler object. With that overwrite,
+ // we have to keep the fwdptr initialized and pointing to our (stale) copy.
+ assert(size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size");
fill_with_object(copy, size);
shenandoah_assert_correct(nullptr, copy_val);
+ // For non-LAB allocations, the object has already been registered
}
shenandoah_assert_correct(nullptr, result);
return result;
@@ -1226,7 +1334,7 @@ void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
}
}
-void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
+size_t ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
assert(start->is_humongous_start(), "reclaim regions starting with the first one");
oop humongous_obj = cast_to_oop(start->bottom());
@@ -1246,6 +1354,7 @@ void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
region->make_trash_immediate();
}
+ return required_regions;
}
class ShenandoahCheckCleanGCLABClosure : public ThreadClosure {
@@ -1255,6 +1364,12 @@ class ShenandoahCheckCleanGCLABClosure : public ThreadClosure {
PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
assert(gclab->words_remaining() == 0, "GCLAB should not need retirement");
+
+ if (ShenandoahHeap::heap()->mode()->is_generational()) {
+ PLAB* plab = ShenandoahThreadLocalData::plab(thread);
+ assert(plab != nullptr, "PLAB should be initialized for %s", thread->name());
+ assert(plab->words_remaining() == 0, "PLAB should not need retirement");
+ }
}
};
@@ -1270,6 +1385,19 @@ class ShenandoahRetireGCLABClosure : public ThreadClosure {
if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) {
ShenandoahThreadLocalData::set_gclab_size(thread, 0);
}
+
+ if (ShenandoahHeap::heap()->mode()->is_generational()) {
+ PLAB* plab = ShenandoahThreadLocalData::plab(thread);
+ assert(plab != nullptr, "PLAB should be initialized for %s", thread->name());
+
+ // There are two reasons to retire all plabs between old-gen evacuation passes.
+ // 1. We need to make the plab memory parsable by remembered-set scanning.
+ // 2. We need to establish a trustworthy UpdateWaterMark value within each old-gen heap region
+ ShenandoahGenerationalHeap::heap()->retire_plab(plab, thread);
+ if (_resize && ShenandoahThreadLocalData::plab_size(thread) > 0) {
+ ShenandoahThreadLocalData::set_plab_size(thread, 0);
+ }
+ }
}
};
@@ -1403,6 +1531,46 @@ void ShenandoahHeap::print_tracing_info() const {
}
}
+void ShenandoahHeap::set_gc_generation(ShenandoahGeneration* generation) {
+ shenandoah_assert_control_or_vm_thread_at_safepoint();
+ _gc_generation = generation;
+}
+
+// Active generation may only be set by the VM thread at a safepoint.
+void ShenandoahHeap::set_active_generation() {
+ assert(Thread::current()->is_VM_thread(), "Only the VM Thread");
+ assert(SafepointSynchronize::is_at_safepoint(), "Only at a safepoint!");
+ assert(_gc_generation != nullptr, "Will set _active_generation to nullptr");
+ _active_generation = _gc_generation;
+}
+
+void ShenandoahHeap::on_cycle_start(GCCause::Cause cause, ShenandoahGeneration* generation) {
+ shenandoah_policy()->record_collection_cause(cause);
+
+ assert(gc_cause() == GCCause::_no_gc, "Over-writing cause");
+ assert(_gc_generation == nullptr, "Over-writing _gc_generation");
+
+ set_gc_cause(cause);
+ set_gc_generation(generation);
+
+ generation->heuristics()->record_cycle_start();
+}
+
+void ShenandoahHeap::on_cycle_end(ShenandoahGeneration* generation) {
+ assert(gc_cause() != GCCause::_no_gc, "cause wasn't set");
+ assert(_gc_generation != nullptr, "_gc_generation wasn't set");
+
+ generation->heuristics()->record_cycle_end();
+ if (mode()->is_generational() && generation->is_global()) {
+ // If we just completed a GLOBAL GC, claim credit for completion of young-gen and old-gen GC as well
+ young_generation()->heuristics()->record_cycle_end();
+ old_generation()->heuristics()->record_cycle_end();
+ }
+
+ set_gc_generation(nullptr);
+ set_gc_cause(GCCause::_no_gc);
+}
+
void ShenandoahHeap::verify(VerifyOption vo) {
if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
if (ShenandoahVerify) {
@@ -1752,107 +1920,11 @@ void ShenandoahHeap::recycle_trash() {
free_set()->recycle_trash();
}
-class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
-private:
- ShenandoahMarkingContext* const _ctx;
-public:
- ShenandoahResetUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
-
- void heap_region_do(ShenandoahHeapRegion* r) {
- if (r->is_active()) {
- // Reset live data and set TAMS optimistically. We would recheck these under the pause
- // anyway to capture any updates that happened since now.
- r->clear_live_data();
- _ctx->capture_top_at_mark_start(r);
- }
- }
-
- bool is_thread_safe() { return true; }
-};
-
-void ShenandoahHeap::prepare_gc() {
- reset_mark_bitmap();
-
- ShenandoahResetUpdateRegionStateClosure cl;
- parallel_heap_region_iterate(&cl);
-}
-
-class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
-private:
- ShenandoahMarkingContext* const _ctx;
- ShenandoahHeapLock* const _lock;
-
-public:
- ShenandoahFinalMarkUpdateRegionStateClosure() :
- _ctx(ShenandoahHeap::heap()->complete_marking_context()), _lock(ShenandoahHeap::heap()->lock()) {}
-
- void heap_region_do(ShenandoahHeapRegion* r) {
- if (r->is_active()) {
- // All allocations past TAMS are implicitly live, adjust the region data.
- // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
- HeapWord *tams = _ctx->top_at_mark_start(r);
- HeapWord *top = r->top();
- if (top > tams) {
- r->increase_live_data_alloc_words(pointer_delta(top, tams));
- }
-
- // We are about to select the collection set, make sure it knows about
- // current pinning status. Also, this allows trashing more regions that
- // now have their pinning status dropped.
- if (r->is_pinned()) {
- if (r->pin_count() == 0) {
- ShenandoahHeapLocker locker(_lock);
- r->make_unpinned();
- }
- } else {
- if (r->pin_count() > 0) {
- ShenandoahHeapLocker locker(_lock);
- r->make_pinned();
- }
- }
-
- // Remember limit for updating refs. It's guaranteed that we get no
- // from-space-refs written from here on.
- r->set_update_watermark_at_safepoint(r->top());
- } else {
- assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
- assert(_ctx->top_at_mark_start(r) == r->top(),
- "Region " SIZE_FORMAT " should have correct TAMS", r->index());
- }
- }
-
- bool is_thread_safe() { return true; }
-};
-
-void ShenandoahHeap::prepare_regions_and_collection_set(bool concurrent) {
- assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
- {
- ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states :
- ShenandoahPhaseTimings::degen_gc_final_update_region_states);
- ShenandoahFinalMarkUpdateRegionStateClosure cl;
- parallel_heap_region_iterate(&cl);
-
- assert_pinned_region_status();
- }
-
- {
- ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset :
- ShenandoahPhaseTimings::degen_gc_choose_cset);
- ShenandoahHeapLocker locker(lock());
- _collection_set->clear();
- heuristics()->choose_collection_set(_collection_set);
- }
-
- {
- ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset :
- ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset);
- ShenandoahHeapLocker locker(lock());
- _free_set->rebuild();
- }
-}
-
void ShenandoahHeap::do_class_unloading() {
_unloader.unload();
+ if (mode()->is_generational()) {
+ old_generation()->set_parsable(false);
+ }
}
void ShenandoahHeap::stw_weak_refs(bool full_gc) {
@@ -1861,7 +1933,8 @@ void ShenandoahHeap::stw_weak_refs(bool full_gc) {
: ShenandoahPhaseTimings::degen_gc_weakrefs;
ShenandoahTimingsTracker t(phase);
ShenandoahGCWorkerPhase worker_phase(phase);
- ref_processor()->process_references(phase, workers(), false /* concurrent */);
+ shenandoah_assert_generations_reconciled();
+ gc_generation()->ref_processor()->process_references(phase, workers(), false /* concurrent */);
}
void ShenandoahHeap::prepare_update_heap_references(bool concurrent) {
@@ -1897,10 +1970,58 @@ void ShenandoahHeap::set_gc_state(uint mask, bool value) {
_gc_state_changed = true;
}
-void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
- assert(!has_forwarded_objects(), "Not expected before/after mark phase");
- set_gc_state(MARKING, in_progress);
- ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
+void ShenandoahHeap::set_concurrent_young_mark_in_progress(bool in_progress) {
+ uint mask;
+ assert(!has_forwarded_objects(), "Young marking is not concurrent with evacuation");
+ if (!in_progress && is_concurrent_old_mark_in_progress()) {
+ assert(mode()->is_generational(), "Only generational GC has old marking");
+ assert(_gc_state.is_set(MARKING), "concurrent_old_marking_in_progress implies MARKING");
+ // If old-marking is in progress when we turn off YOUNG_MARKING, leave MARKING (and OLD_MARKING) on
+ mask = YOUNG_MARKING;
+ } else {
+ mask = MARKING | YOUNG_MARKING;
+ }
+ set_gc_state(mask, in_progress);
+ manage_satb_barrier(in_progress);
+}
+
+void ShenandoahHeap::set_concurrent_old_mark_in_progress(bool in_progress) {
+#ifdef ASSERT
+ // has_forwarded_objects() iff UPDATEREFS or EVACUATION
+ bool has_forwarded = has_forwarded_objects();
+ bool updating_or_evacuating = _gc_state.is_set(UPDATEREFS | EVACUATION);
+ bool evacuating = _gc_state.is_set(EVACUATION);
+ assert ((has_forwarded == updating_or_evacuating) || (evacuating && !has_forwarded && collection_set()->is_empty()),
+ "Updating or evacuating iff has forwarded objects, or if evacuation phase is promoting in place without forwarding");
+#endif
+ if (!in_progress && is_concurrent_young_mark_in_progress()) {
+ // If young-marking is in progress when we turn off OLD_MARKING, leave MARKING (and YOUNG_MARKING) on
+ assert(_gc_state.is_set(MARKING), "concurrent_young_marking_in_progress implies MARKING");
+ set_gc_state(OLD_MARKING, in_progress);
+ } else {
+ set_gc_state(MARKING | OLD_MARKING, in_progress);
+ }
+ manage_satb_barrier(in_progress);
+}
+
+bool ShenandoahHeap::is_prepare_for_old_mark_in_progress() const {
+ return old_generation()->is_preparing_for_mark();
+}
+
+void ShenandoahHeap::manage_satb_barrier(bool active) {
+ if (is_concurrent_mark_in_progress()) {
+ // Ignore request to deactivate barrier while concurrent mark is in progress.
+ // Do not attempt to re-activate the barrier if it is already active.
+ if (active && !ShenandoahBarrierSet::satb_mark_queue_set().is_active()) {
+ ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(active, !active);
+ }
+ } else {
+ // No concurrent marking is in progress so honor request to deactivate,
+ // but only if the barrier is already active.
+ if (!active && ShenandoahBarrierSet::satb_mark_queue_set().is_active()) {
+ ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(active, !active);
+ }
+ }
}
void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
@@ -1933,11 +2054,23 @@ bool ShenandoahHeap::try_cancel_gc() {
return prev == CANCELLABLE;
}
+void ShenandoahHeap::cancel_concurrent_mark() {
+ if (mode()->is_generational()) {
+ young_generation()->cancel_marking();
+ old_generation()->cancel_marking();
+ }
+
+ global_generation()->cancel_marking();
+
+ ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking();
+}
+
void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
if (try_cancel_gc()) {
FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
log_info(gc)("%s", msg.buffer());
Events::log(Thread::current(), "%s", msg.buffer());
+ _cancel_requested_time = os::elapsedTime();
}
}
@@ -2061,12 +2194,13 @@ address ShenandoahHeap::in_cset_fast_test_addr() {
return (address) heap->collection_set()->biased_map_address();
}
-size_t ShenandoahHeap::bytes_allocated_since_gc_start() const {
- return Atomic::load(&_bytes_allocated_since_gc_start);
-}
-
void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
- Atomic::store(&_bytes_allocated_since_gc_start, (size_t)0);
+ if (mode()->is_generational()) {
+ young_generation()->reset_bytes_allocated_since_gc_start();
+ old_generation()->reset_bytes_allocated_since_gc_start();
+ }
+
+ global_generation()->reset_bytes_allocated_since_gc_start();
}
void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
@@ -2130,8 +2264,11 @@ void ShenandoahHeap::sync_pinned_region_status() {
void ShenandoahHeap::assert_pinned_region_status() {
for (size_t i = 0; i < num_regions(); i++) {
ShenandoahHeapRegion* r = get_region(i);
- assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
- "Region " SIZE_FORMAT " pinning status is inconsistent", i);
+ shenandoah_assert_generations_reconciled();
+ if (gc_generation()->contains(r)) {
+ assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
+ "Region " SIZE_FORMAT " pinning status is inconsistent", i);
+ }
}
}
#endif
@@ -2186,7 +2323,7 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask {
ShenandoahHeap* _heap;
ShenandoahRegionIterator* _regions;
public:
- ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions) :
+ explicit ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions) :
WorkerTask("Shenandoah Update References"),
_heap(ShenandoahHeap::heap()),
_regions(regions) {
@@ -2206,27 +2343,28 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask {
private:
template
void do_work(uint worker_id) {
- T cl;
if (CONCURRENT && (worker_id == 0)) {
// We ask the first worker to replenish the Mutator free set by moving regions previously reserved to hold the
// results of evacuation. These reserves are no longer necessary because evacuation has completed.
size_t cset_regions = _heap->collection_set()->count();
- // We cannot transfer any more regions than will be reclaimed when the existing collection set is recycled because
- // we need the reclaimed collection set regions to replenish the collector reserves
+
+ // Now that evacuation is done, we can reassign any regions that had been reserved to hold the results of evacuation
+ // to the mutator free set. At the end of GC, we will have cset_regions newly evacuated fully empty regions from
+ // which we will be able to replenish the Collector free set and the OldCollector free set in preparation for the
+ // next GC cycle.
_heap->free_set()->move_regions_from_collector_to_mutator(cset_regions);
}
// If !CONCURRENT, there's no value in expanding Mutator free set
-
+ T cl;
ShenandoahHeapRegion* r = _regions->next();
- ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
while (r != nullptr) {
HeapWord* update_watermark = r->get_update_watermark();
assert (update_watermark >= r->bottom(), "sanity");
if (r->is_active() && !r->is_cset()) {
_heap->marked_object_oop_iterate(r, &cl, update_watermark);
- }
- if (ShenandoahPacing) {
- _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
+ if (ShenandoahPacing) {
+ _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
+ }
}
if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) {
return;
@@ -2248,36 +2386,6 @@ void ShenandoahHeap::update_heap_references(bool concurrent) {
}
}
-
-class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
-private:
- ShenandoahHeapLock* const _lock;
-
-public:
- ShenandoahFinalUpdateRefsUpdateRegionStateClosure() : _lock(ShenandoahHeap::heap()->lock()) {}
-
- void heap_region_do(ShenandoahHeapRegion* r) {
- // Drop unnecessary "pinned" state from regions that does not have CP marks
- // anymore, as this would allow trashing them.
-
- if (r->is_active()) {
- if (r->is_pinned()) {
- if (r->pin_count() == 0) {
- ShenandoahHeapLocker locker(_lock);
- r->make_unpinned();
- }
- } else {
- if (r->pin_count() > 0) {
- ShenandoahHeapLocker locker(_lock);
- r->make_pinned();
- }
- }
- }
- }
-
- bool is_thread_safe() { return true; }
-};
-
void ShenandoahHeap::update_heap_region_states(bool concurrent) {
assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
@@ -2286,8 +2394,8 @@ void ShenandoahHeap::update_heap_region_states(bool concurrent) {
ShenandoahGCPhase phase(concurrent ?
ShenandoahPhaseTimings::final_update_refs_update_region_states :
ShenandoahPhaseTimings::degen_gc_final_update_refs_update_region_states);
- ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl;
- parallel_heap_region_iterate(&cl);
+
+ final_update_refs_update_region_states();
assert_pinned_region_status();
}
@@ -2300,13 +2408,54 @@ void ShenandoahHeap::update_heap_region_states(bool concurrent) {
}
}
+void ShenandoahHeap::final_update_refs_update_region_states() {
+ ShenandoahSynchronizePinnedRegionStates cl;
+ parallel_heap_region_iterate(&cl);
+}
+
void ShenandoahHeap::rebuild_free_set(bool concurrent) {
- {
- ShenandoahGCPhase phase(concurrent ?
- ShenandoahPhaseTimings::final_update_refs_rebuild_freeset :
- ShenandoahPhaseTimings::degen_gc_final_update_refs_rebuild_freeset);
- ShenandoahHeapLocker locker(lock());
- _free_set->rebuild();
+ ShenandoahGCPhase phase(concurrent ?
+ ShenandoahPhaseTimings::final_update_refs_rebuild_freeset :
+ ShenandoahPhaseTimings::degen_gc_final_update_refs_rebuild_freeset);
+ ShenandoahHeapLocker locker(lock());
+ size_t young_cset_regions, old_cset_regions;
+ size_t first_old_region, last_old_region, old_region_count;
+ _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old_region, last_old_region, old_region_count);
+ // If there are no old regions, first_old_region will be greater than last_old_region
+ assert((first_old_region > last_old_region) ||
+ ((last_old_region + 1 - first_old_region >= old_region_count) &&
+ get_region(first_old_region)->is_old() && get_region(last_old_region)->is_old()),
+ "sanity: old_region_count: " SIZE_FORMAT ", first_old_region: " SIZE_FORMAT ", last_old_region: " SIZE_FORMAT,
+ old_region_count, first_old_region, last_old_region);
+
+ if (mode()->is_generational()) {
+#ifdef ASSERT
+ if (ShenandoahVerify) {
+ verifier()->verify_before_rebuilding_free_set();
+ }
+#endif
+
+ // The computation of bytes_of_allocation_runway_before_gc_trigger is quite conservative so consider all of this
+ // available for transfer to old. Note that transfer of humongous regions does not impact available.
+ ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap();
+ size_t allocation_runway = gen_heap->young_generation()->heuristics()->bytes_of_allocation_runway_before_gc_trigger(young_cset_regions);
+ gen_heap->compute_old_generation_balance(allocation_runway, old_cset_regions);
+
+ // Total old_available may have been expanded to hold anticipated promotions. We trigger if the fragmented available
+ // memory represents more than 16 regions worth of data. Note that fragmentation may increase when we promote regular
+ // regions in place when many of these regular regions have an abundant amount of available memory within them. Fragmentation
+ // will decrease as promote-by-copy consumes the available memory within these partially consumed regions.
+ //
+ // We consider old-gen to have excessive fragmentation if more than 12.5% of old-gen is free memory that resides
+ // within partially consumed regions of memory.
+ }
+ // Rebuild free set based on adjusted generation sizes.
+ _free_set->finish_rebuild(young_cset_regions, old_cset_regions, old_region_count);
+
+ if (mode()->is_generational()) {
+ ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap();
+ ShenandoahOldGeneration* old_gen = gen_heap->old_generation();
+ old_gen->heuristics()->evaluate_triggers(first_old_region, last_old_region, old_region_count, num_regions());
}
}
@@ -2429,7 +2578,7 @@ GrowableArray ShenandoahHeap::memory_pools() {
}
MemoryUsage ShenandoahHeap::memory_usage() {
- return _memory_pool->get_memory_usage();
+ return MemoryUsage(_initial_size, used(), committed(), max_capacity());
}
ShenandoahRegionIterator::ShenandoahRegionIterator() :
@@ -2571,3 +2720,26 @@ void ShenandoahHeap::complete_loaded_archive_space(MemRegion archive_space) {
p2i(end), p2i(end_reg->top()));
#endif
}
+
+ShenandoahGeneration* ShenandoahHeap::generation_for(ShenandoahAffiliation affiliation) const {
+ if (!mode()->is_generational()) {
+ return global_generation();
+ } else if (affiliation == YOUNG_GENERATION) {
+ return young_generation();
+ } else if (affiliation == OLD_GENERATION) {
+ return old_generation();
+ }
+
+ ShouldNotReachHere();
+ return nullptr;
+}
+
+void ShenandoahHeap::log_heap_status(const char* msg) const {
+ if (mode()->is_generational()) {
+ young_generation()->log_status(msg);
+ old_generation()->log_status(msg);
+ } else {
+ global_generation()->log_status(msg);
+ }
+}
+
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
index 7e616f925d0..a9a793f9e60 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -29,11 +30,16 @@
#include "gc/shared/markBitMap.hpp"
#include "gc/shared/softRefPolicy.hpp"
#include "gc/shared/collectedHeap.hpp"
-#include "gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp"
-#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahAllocRequest.hpp"
+#include "gc/shenandoah/shenandoahAsserts.hpp"
+#include "gc/shenandoah/shenandoahController.hpp"
#include "gc/shenandoah/shenandoahLock.hpp"
#include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
+#include "gc/shenandoah/shenandoahEvacTracker.hpp"
+#include "gc/shenandoah/shenandoahGenerationType.hpp"
+#include "gc/shenandoah/shenandoahGenerationSizer.hpp"
+#include "gc/shenandoah/shenandoahMmuTracker.hpp"
+#include "gc/shenandoah/mode/shenandoahMode.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
#include "gc/shenandoah/shenandoahSharedVariables.hpp"
#include "gc/shenandoah/shenandoahUnload.hpp"
@@ -45,9 +51,11 @@
class ConcurrentGCTimer;
class ObjectIterateScanRootClosure;
class ShenandoahCollectorPolicy;
-class ShenandoahControlThread;
class ShenandoahGCSession;
class ShenandoahGCStateResetter;
+class ShenandoahGeneration;
+class ShenandoahYoungGeneration;
+class ShenandoahOldGeneration;
class ShenandoahHeuristics;
class ShenandoahMarkingContext;
class ShenandoahMode;
@@ -117,7 +125,7 @@ typedef Stack ShenandoahScanObjectStack;
// to encode forwarding data. See BrooksPointer for details on forwarding data encoding.
// See ShenandoahControlThread for GC cycle structure.
//
-class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
+class ShenandoahHeap : public CollectedHeap {
friend class ShenandoahAsserts;
friend class VMStructs;
friend class ShenandoahGCSession;
@@ -127,6 +135,7 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
// Supported GC
friend class ShenandoahConcurrentGC;
+ friend class ShenandoahOldGC;
friend class ShenandoahDegenGC;
friend class ShenandoahFullGC;
friend class ShenandoahUnload;
@@ -136,11 +145,46 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
private:
ShenandoahHeapLock _lock;
+ // Indicates the generation whose collection is in
+ // progress. Mutator threads aren't allowed to read
+ // this field.
+ ShenandoahGeneration* _gc_generation;
+
+ // This is set and cleared by only the VMThread
+ // at each STW pause (safepoint) to the value seen in
+ // _gc_generation. This allows the value to be always consistently
+ // seen by all mutators as well as all GC worker threads.
+ // In that sense, it's a stable snapshot of _gc_generation that is
+ // updated at each STW pause associated with a ShenandoahVMOp.
+ ShenandoahGeneration* _active_generation;
+
public:
ShenandoahHeapLock* lock() {
return &_lock;
}
+ ShenandoahGeneration* gc_generation() const {
+ // We don't want this field read by a mutator thread
+ assert(!Thread::current()->is_Java_thread(), "Not allowed");
+ // value of _gc_generation field, see above
+ return _gc_generation;
+ }
+
+ ShenandoahGeneration* active_generation() const {
+ // value of _active_generation field, see above
+ return _active_generation;
+ }
+
+ // Set the _gc_generation field
+ void set_gc_generation(ShenandoahGeneration* generation);
+
+ // Copy the value in the _gc_generation field into
+ // the _active_generation field: can only be called at
+ // a safepoint by the VMThread.
+ void set_active_generation();
+
+ ShenandoahHeuristics* heuristics();
+
// ---------- Initialization, termination, identification, printing routines
//
public:
@@ -153,8 +197,8 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
jint initialize() override;
void post_initialize() override;
void initialize_mode();
- void initialize_heuristics();
-
+ virtual void initialize_heuristics();
+ virtual void print_init_logger() const;
void initialize_serviceability() override;
void print_on(outputStream* st) const override;
@@ -175,35 +219,34 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
// ---------- Heap counters and metrics
//
private:
- size_t _initial_size;
- size_t _minimum_size;
+ size_t _initial_size;
+ size_t _minimum_size;
+
volatile size_t _soft_max_size;
shenandoah_padding(0);
- volatile size_t _used;
volatile size_t _committed;
- volatile size_t _bytes_allocated_since_gc_start;
shenandoah_padding(1);
+ void increase_used(const ShenandoahAllocRequest& req);
+
public:
- void increase_used(size_t bytes);
- void decrease_used(size_t bytes);
- void set_used(size_t bytes);
+ void increase_used(ShenandoahGeneration* generation, size_t bytes);
+ void decrease_used(ShenandoahGeneration* generation, size_t bytes);
+ void increase_humongous_waste(ShenandoahGeneration* generation, size_t bytes);
+ void decrease_humongous_waste(ShenandoahGeneration* generation, size_t bytes);
void increase_committed(size_t bytes);
void decrease_committed(size_t bytes);
- void increase_allocated(size_t bytes);
- size_t bytes_allocated_since_gc_start() const override;
void reset_bytes_allocated_since_gc_start();
size_t min_capacity() const;
size_t max_capacity() const override;
- size_t soft_max_capacity() const override;
+ size_t soft_max_capacity() const;
size_t initial_capacity() const;
size_t capacity() const override;
size_t used() const override;
size_t committed() const;
- size_t available() const override;
void set_soft_max_capacity(size_t v);
@@ -223,6 +266,8 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
ShenandoahWorkerThreads* _workers;
ShenandoahWorkerThreads* _safepoint_workers;
+ virtual void initialize_controller();
+
public:
uint max_workers();
void assert_gc_workers(uint nworker) NOT_DEBUG_RETURN;
@@ -239,7 +284,7 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
bool _heap_region_special;
size_t _num_regions;
ShenandoahHeapRegion** _regions;
- ShenandoahRegionIterator _update_refs_iterator;
+ uint8_t* _affiliations; // Holds array of enum ShenandoahAffiliation, including FREE status in non-generational mode
public:
@@ -256,6 +301,8 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
void heap_region_iterate(ShenandoahHeapRegionClosure* blk) const;
void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const;
+ inline ShenandoahMmuTracker* mmu_tracker() { return &_mmu_tracker; };
+
// ---------- GC state machinery
//
// GC state describes the important parts of collector state, that may be
@@ -271,6 +318,7 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
HAS_FORWARDED_BITPOS = 0,
// Heap is under marking: needs SATB barriers.
+ // For generational mode, it means either young or old marking, or both.
MARKING_BITPOS = 1,
// Heap is under evacuation: needs LRB barriers. (Set together with HAS_FORWARDED)
@@ -281,6 +329,12 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
// Heap is under weak-reference/roots processing: needs weak-LRB barriers.
WEAK_ROOTS_BITPOS = 4,
+
+ // Young regions are under marking, need SATB barriers.
+ YOUNG_MARKING_BITPOS = 5,
+
+ // Old regions are under marking, need SATB barriers.
+ OLD_MARKING_BITPOS = 6
};
enum GCState {
@@ -290,13 +344,13 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
EVACUATION = 1 << EVACUATION_BITPOS,
UPDATEREFS = 1 << UPDATEREFS_BITPOS,
WEAK_ROOTS = 1 << WEAK_ROOTS_BITPOS,
+ YOUNG_MARKING = 1 << YOUNG_MARKING_BITPOS,
+ OLD_MARKING = 1 << OLD_MARKING_BITPOS
};
private:
bool _gc_state_changed;
ShenandoahSharedBitmap _gc_state;
-
- // tracks if new regions have been allocated or retired since last check
ShenandoahSharedFlag _heap_changed;
ShenandoahSharedFlag _degenerated_gc_in_progress;
ShenandoahSharedFlag _full_gc_in_progress;
@@ -325,7 +379,8 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
return _heap_changed.try_unset();
}
- void set_concurrent_mark_in_progress(bool in_progress);
+ void set_concurrent_young_mark_in_progress(bool in_progress);
+ void set_concurrent_old_mark_in_progress(bool in_progress);
void set_evacuation_in_progress(bool in_progress);
void set_update_refs_in_progress(bool in_progress);
void set_degenerated_gc_in_progress(bool in_progress);
@@ -337,7 +392,10 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
inline bool is_stable() const;
inline bool is_idle() const;
+
inline bool is_concurrent_mark_in_progress() const;
+ inline bool is_concurrent_young_mark_in_progress() const;
+ inline bool is_concurrent_old_mark_in_progress() const;
inline bool is_update_refs_in_progress() const;
inline bool is_evacuation_in_progress() const;
inline bool is_degenerated_gc_in_progress() const;
@@ -348,8 +406,11 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
inline bool is_stw_gc_in_progress() const;
inline bool is_concurrent_strong_root_in_progress() const;
inline bool is_concurrent_weak_root_in_progress() const;
+ bool is_prepare_for_old_mark_in_progress() const;
private:
+ void manage_satb_barrier(bool active);
+
enum CancelState {
// Normal state. GC has not been cancelled and is open for cancellation.
// Worker threads can suspend for safepoint.
@@ -360,16 +421,22 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
CANCELLED
};
+ double _cancel_requested_time;
ShenandoahSharedEnumFlag _cancelled_gc;
+
+ // Returns true if cancel request was successfully communicated.
+ // Returns false if some other thread already communicated cancel
+ // request. A true return value does not mean GC has been
+ // cancelled, only that the process of cancelling GC has begun.
bool try_cancel_gc();
public:
-
inline bool cancelled_gc() const;
inline bool check_cancelled_gc_and_yield(bool sts_active = true);
- inline void clear_cancelled_gc();
+ inline void clear_cancelled_gc(bool clear_oom_handler = true);
+ void cancel_concurrent_mark();
void cancel_gc(GCCause::Cause cause);
public:
@@ -381,13 +448,16 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
// Returns true if the soft maximum heap has been changed using management APIs.
bool check_soft_max_changed();
+protected:
+ // This is shared between shConcurrentGC and shDegenerateGC so that degenerated
+ // GC can resume update refs from where the concurrent GC was cancelled. It is
+ // also used in shGenerationalHeap, which uses a different closure for update refs.
+ ShenandoahRegionIterator _update_refs_iterator;
+
private:
// GC support
- // Reset bitmap, prepare regions for new GC cycle
- void prepare_gc();
- void prepare_regions_and_collection_set(bool concurrent);
// Evacuation
- void evacuate_collection_set(bool concurrent);
+ virtual void evacuate_collection_set(bool concurrent);
// Concurrent root processing
void prepare_concurrent_roots();
void finish_concurrent_roots();
@@ -395,14 +465,15 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
void do_class_unloading();
// Reference updating
void prepare_update_heap_references(bool concurrent);
- void update_heap_references(bool concurrent);
+ virtual void update_heap_references(bool concurrent);
// Final update region states
void update_heap_region_states(bool concurrent);
- void rebuild_free_set(bool concurrent);
+ virtual void final_update_refs_update_region_states();
void rendezvous_threads(const char* name);
void recycle_trash();
public:
+ void rebuild_free_set(bool concurrent);
void notify_gc_progress();
void notify_gc_no_progress();
size_t get_gc_no_progress_count() const;
@@ -410,27 +481,52 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
//
// Mark support
private:
- ShenandoahControlThread* _control_thread;
+ ShenandoahGeneration* _global_generation;
+
+protected:
+ ShenandoahController* _control_thread;
+
+ ShenandoahYoungGeneration* _young_generation;
+ ShenandoahOldGeneration* _old_generation;
+
+private:
ShenandoahCollectorPolicy* _shenandoah_policy;
ShenandoahMode* _gc_mode;
- ShenandoahHeuristics* _heuristics;
ShenandoahFreeSet* _free_set;
ShenandoahPacer* _pacer;
ShenandoahVerifier* _verifier;
- ShenandoahPhaseTimings* _phase_timings;
-
- ShenandoahControlThread* control_thread() { return _control_thread; }
+ ShenandoahPhaseTimings* _phase_timings;
+ ShenandoahMmuTracker _mmu_tracker;
public:
+ ShenandoahController* control_thread() { return _control_thread; }
+
+ ShenandoahGeneration* global_generation() const { return _global_generation; }
+ ShenandoahYoungGeneration* young_generation() const {
+ assert(mode()->is_generational(), "Young generation requires generational mode");
+ return _young_generation;
+ }
+
+ ShenandoahOldGeneration* old_generation() const {
+ assert(mode()->is_generational(), "Old generation requires generational mode");
+ return _old_generation;
+ }
+
+ ShenandoahGeneration* generation_for(ShenandoahAffiliation affiliation) const;
+
ShenandoahCollectorPolicy* shenandoah_policy() const { return _shenandoah_policy; }
ShenandoahMode* mode() const { return _gc_mode; }
- ShenandoahHeuristics* heuristics() const { return _heuristics; }
ShenandoahFreeSet* free_set() const { return _free_set; }
ShenandoahPacer* pacer() const { return _pacer; }
ShenandoahPhaseTimings* phase_timings() const { return _phase_timings; }
+ ShenandoahEvacOOMHandler* oom_evac_handler() { return &_oom_evac_handler; }
+
+ void on_cycle_start(GCCause::Cause cause, ShenandoahGeneration* generation);
+ void on_cycle_end(ShenandoahGeneration* generation);
+
ShenandoahVerifier* verifier();
// ---------- VM subsystem bindings
@@ -444,7 +540,7 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
// For exporting to SA
int _log_min_obj_alignment_in_bytes;
public:
- ShenandoahMonitoringSupport* monitoring_support() { return _monitoring_support; }
+ ShenandoahMonitoringSupport* monitoring_support() const { return _monitoring_support; }
GCMemoryManager* cycle_memory_manager() { return &_cycle_memory_manager; }
GCMemoryManager* stw_memory_manager() { return &_stw_memory_manager; }
@@ -454,14 +550,6 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
GCTracer* tracer();
ConcurrentGCTimer* gc_timer() const;
-// ---------- Reference processing
-//
-private:
- ShenandoahReferenceProcessor* const _ref_processor;
-
-public:
- ShenandoahReferenceProcessor* ref_processor() { return _ref_processor; }
-
// ---------- Class Unloading
//
private:
@@ -480,6 +568,9 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
void stw_process_weak_roots(bool full_gc);
void stw_weak_refs(bool full_gc);
+ inline void assert_lock_for_affiliation(ShenandoahAffiliation orig_affiliation,
+ ShenandoahAffiliation new_affiliation);
+
// Heap iteration support
void scan_roots_for_iteration(ShenandoahScanObjectStack* oop_stack, ObjectIterateScanRootClosure* oops);
bool prepare_aux_bitmap_for_iteration();
@@ -497,6 +588,21 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
// Use is_in_reserved to check if object is within heap bounds.
bool is_in(const void* p) const override;
+ // Returns true if the given oop belongs to a generation that is actively being collected.
+ inline bool is_in_active_generation(oop obj) const;
+ inline bool is_in_young(const void* p) const;
+ inline bool is_in_old(const void* p) const;
+
+ // Returns true iff the young generation is being collected and the given pointer
+ // is in the old generation. This is used to prevent the young collection from treating
+ // such an object as unreachable.
+ inline bool is_in_old_during_young_collection(oop obj) const;
+
+ inline ShenandoahAffiliation region_affiliation(const ShenandoahHeapRegion* r);
+ inline void set_affiliation(ShenandoahHeapRegion* r, ShenandoahAffiliation new_affiliation);
+
+ inline ShenandoahAffiliation region_affiliation(size_t index);
+
bool requires_barriers(stackChunkOop obj) const override;
MemRegion reserved_region() const { return _reserved; }
@@ -543,15 +649,17 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
// ---------- CDS archive support
- bool can_load_archived_objects() const override { return true; }
+ bool can_load_archived_objects() const override { return !ShenandoahCardBarrier; }
HeapWord* allocate_loaded_archive_space(size_t size) override;
void complete_loaded_archive_space(MemRegion archive_space) override;
// ---------- Allocation support
//
+protected:
+ inline HeapWord* allocate_from_gclab(Thread* thread, size_t size);
+
private:
HeapWord* allocate_memory_under_lock(ShenandoahAllocRequest& request, bool& in_new_region);
- inline HeapWord* allocate_from_gclab(Thread* thread, size_t size);
HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size);
HeapWord* allocate_new_gclab(size_t min_size, size_t word_size, size_t* actual_size);
@@ -562,7 +670,7 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
size_t size,
Metaspace::MetadataType mdtype) override;
- void notify_mutator_alloc_words(size_t words, bool waste);
+ void notify_mutator_alloc_words(size_t words, size_t waste);
HeapWord* allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) override;
size_t tlab_capacity(Thread *thr) const override;
@@ -600,8 +708,6 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
public:
inline ShenandoahMarkingContext* complete_marking_context() const;
inline ShenandoahMarkingContext* marking_context() const;
- inline void mark_complete_marking_context();
- inline void mark_incomplete_marking_context();
template
inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl);
@@ -612,8 +718,6 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
template
inline void marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
- void reset_mark_bitmap();
-
// SATB barriers hooks
inline bool requires_marking(const void* entry) const;
@@ -634,6 +738,8 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
ShenandoahCollectionSet* _collection_set;
ShenandoahEvacOOMHandler _oom_evac_handler;
+ oop try_evacuate_object(oop src, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahAffiliation target_gen);
+
public:
static address in_cset_fast_test_addr();
@@ -645,9 +751,9 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
// Checks if location is in the collection set. Can be interior pointer, not the oop itself.
inline bool in_collection_set_loc(void* loc) const;
- // Evacuates object src. Returns the evacuated object, either evacuated
+ // Evacuates or promotes object src. Returns the evacuated object, either evacuated
// by this thread, or by some other thread.
- oop evacuate_object(oop src, Thread* thread);
+ virtual oop evacuate_object(oop src, Thread* thread);
// Call before/after evacuation.
inline void enter_evacuation(Thread* t);
@@ -674,7 +780,16 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
static inline void atomic_clear_oop(narrowOop* addr, oop compare);
static inline void atomic_clear_oop(narrowOop* addr, narrowOop compare);
- void trash_humongous_region_at(ShenandoahHeapRegion *r);
+ size_t trash_humongous_region_at(ShenandoahHeapRegion *r);
+
+ static inline void increase_object_age(oop obj, uint additional_age);
+
+ // Return the object's age, or a sentinel value when the age can't
+ // necessarily be determined because of concurrent locking by the
+ // mutator
+ static inline uint get_object_age(oop obj);
+
+ void log_heap_status(const char *msg) const;
private:
void trash_cset_regions();
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
index 6131d079590..2b34b2d5cfc 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2015, 2020, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -40,14 +41,16 @@
#include "gc/shenandoah/shenandoahWorkGroup.hpp"
#include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
-#include "gc/shenandoah/shenandoahControlThread.hpp"
+#include "gc/shenandoah/shenandoahGeneration.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "gc/shenandoah/mode/shenandoahMode.hpp"
#include "oops/compressedOops.inline.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/atomic.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/prefetch.inline.hpp"
+#include "runtime/objectMonitor.inline.hpp"
#include "utilities/copy.hpp"
#include "utilities/globalDefinitions.hpp"
@@ -264,9 +267,16 @@ inline bool ShenandoahHeap::check_cancelled_gc_and_yield(bool sts_active) {
return cancelled_gc();
}
-inline void ShenandoahHeap::clear_cancelled_gc() {
+inline void ShenandoahHeap::clear_cancelled_gc(bool clear_oom_handler) {
_cancelled_gc.set(CANCELLABLE);
- _oom_evac_handler.clear();
+ if (_cancel_requested_time > 0) {
+ log_debug(gc)("GC cancellation took %.3fs", (os::elapsedTime() - _cancel_requested_time));
+ _cancel_requested_time = 0;
+ }
+
+ if (clear_oom_handler) {
+ _oom_evac_handler.clear();
+ }
}
inline HeapWord* ShenandoahHeap::allocate_from_gclab(Thread* thread, size_t size) {
@@ -283,10 +293,148 @@ inline HeapWord* ShenandoahHeap::allocate_from_gclab(Thread* thread, size_t size
if (obj != nullptr) {
return obj;
}
- // Otherwise...
return allocate_from_gclab_slow(thread, size);
}
+void ShenandoahHeap::increase_object_age(oop obj, uint additional_age) {
+ // This operates on new copy of an object. This means that the object's mark-word
+ // is thread-local and therefore safe to access. However, when the mark is
+ // displaced (i.e. stack-locked or monitor-locked), then it must be considered
+ // a shared memory location. It can be accessed by other threads.
+ // In particular, a competing evacuating thread can succeed to install its copy
+ // as the forwardee and continue to unlock the object, at which point 'our'
+ // write to the foreign stack-location would potentially over-write random
+ // information on that stack. Writing to a monitor is less problematic,
+ // but still not safe: while the ObjectMonitor would not randomly disappear,
+ // the other thread would also write to the same displaced header location,
+ // possibly leading to increase the age twice.
+ // For all these reasons, we take the conservative approach and not attempt
+ // to increase the age when the header is displaced.
+ markWord w = obj->mark();
+ // The mark-word has been copied from the original object. It can not be
+ // inflating, because inflation can not be interrupted by a safepoint,
+ // and after a safepoint, a Java thread would first have to successfully
+ // evacuate the object before it could inflate the monitor.
+ assert(!w.is_being_inflated() || LockingMode == LM_LIGHTWEIGHT, "must not inflate monitor before evacuation of object succeeds");
+ // It is possible that we have copied the object after another thread has
+ // already successfully completed evacuation. While harmless (we would never
+ // publish our copy), don't even attempt to modify the age when that
+ // happens.
+ if (!w.has_displaced_mark_helper() && !w.is_marked()) {
+ w = w.set_age(MIN2(markWord::max_age, w.age() + additional_age));
+ obj->set_mark(w);
+ }
+}
+
+// Return the object's age, or a sentinel value when the age can't
+// necessarily be determined because of concurrent locking by the
+// mutator
+uint ShenandoahHeap::get_object_age(oop obj) {
+ markWord w = obj->mark();
+ assert(!w.is_marked(), "must not be forwarded");
+ if (UseObjectMonitorTable) {
+ assert(LockingMode == LM_LIGHTWEIGHT, "Must use LW locking, too");
+ assert(w.age() <= markWord::max_age, "Impossible!");
+ return w.age();
+ }
+ if (w.has_monitor()) {
+ w = w.monitor()->header();
+ } else if (w.is_being_inflated() || w.has_displaced_mark_helper()) {
+ // Informs caller that we aren't able to determine the age
+ return markWord::max_age + 1; // sentinel
+ }
+ assert(w.age() <= markWord::max_age, "Impossible!");
+ return w.age();
+}
+
+inline bool ShenandoahHeap::is_in_active_generation(oop obj) const {
+ if (!mode()->is_generational()) {
+ // everything is the same single generation
+ assert(is_in_reserved(obj), "Otherwise shouldn't return true below");
+ return true;
+ }
+
+ ShenandoahGeneration* const gen = active_generation();
+
+ if (gen == nullptr) {
+ // no collection is happening: only expect this to be called
+ // when concurrent processing is active, but that could change
+ return false;
+ }
+
+ assert(is_in_reserved(obj), "only check if is in active generation for objects (" PTR_FORMAT ") in heap", p2i(obj));
+ assert(gen->is_old() || gen->is_young() || gen->is_global(),
+ "Active generation must be old, young, or global");
+
+ size_t index = heap_region_containing(obj)->index();
+
+ // No flickering!
+ assert(gen == active_generation(), "Race?");
+
+ switch (_affiliations[index]) {
+ case ShenandoahAffiliation::FREE:
+ // Free regions are in old, young, and global collections
+ return true;
+ case ShenandoahAffiliation::YOUNG_GENERATION:
+ // Young regions are in young and global collections, not in old collections
+ return !gen->is_old();
+ case ShenandoahAffiliation::OLD_GENERATION:
+ // Old regions are in old and global collections, not in young collections
+ return !gen->is_young();
+ default:
+ assert(false, "Bad affiliation (%d) for region " SIZE_FORMAT, _affiliations[index], index);
+ return false;
+ }
+}
+
+inline bool ShenandoahHeap::is_in_young(const void* p) const {
+ return is_in_reserved(p) && (_affiliations[heap_region_index_containing(p)] == ShenandoahAffiliation::YOUNG_GENERATION);
+}
+
+inline bool ShenandoahHeap::is_in_old(const void* p) const {
+ return is_in_reserved(p) && (_affiliations[heap_region_index_containing(p)] == ShenandoahAffiliation::OLD_GENERATION);
+}
+
+inline bool ShenandoahHeap::is_in_old_during_young_collection(oop obj) const {
+ return active_generation()->is_young() && is_in_old(obj);
+}
+
+inline ShenandoahAffiliation ShenandoahHeap::region_affiliation(const ShenandoahHeapRegion *r) {
+ return (ShenandoahAffiliation) _affiliations[r->index()];
+}
+
+inline void ShenandoahHeap::assert_lock_for_affiliation(ShenandoahAffiliation orig_affiliation,
+ ShenandoahAffiliation new_affiliation) {
+ // A lock is required when changing from FREE to NON-FREE. Though it may be possible to elide the lock when
+ // transitioning from in-use to FREE, the current implementation uses a lock for this transition. A lock is
+ // not required to change from YOUNG to OLD (i.e. when promoting humongous region).
+ //
+ // new_affiliation is: FREE YOUNG OLD
+ // orig_affiliation is: FREE X L L
+ // YOUNG L X
+ // OLD L X X
+ // X means state transition won't happen (so don't care)
+ // L means lock should be held
+ // Blank means no lock required because affiliation visibility will not be required until subsequent safepoint
+ //
+ // Note: during full GC, all transitions between states are possible. During Full GC, we should be in a safepoint.
+
+ if ((orig_affiliation == ShenandoahAffiliation::FREE) || (new_affiliation == ShenandoahAffiliation::FREE)) {
+ shenandoah_assert_heaplocked_or_safepoint();
+ }
+}
+
+inline void ShenandoahHeap::set_affiliation(ShenandoahHeapRegion* r, ShenandoahAffiliation new_affiliation) {
+#ifdef ASSERT
+ assert_lock_for_affiliation(region_affiliation(r), new_affiliation);
+#endif
+ _affiliations[r->index()] = (uint8_t) new_affiliation;
+}
+
+inline ShenandoahAffiliation ShenandoahHeap::region_affiliation(size_t index) {
+ return (ShenandoahAffiliation) _affiliations[index];
+}
+
inline bool ShenandoahHeap::requires_marking(const void* entry) const {
oop obj = cast_to_oop(entry);
return !_marking_context->is_marked_strong(obj);
@@ -314,6 +462,14 @@ inline bool ShenandoahHeap::is_concurrent_mark_in_progress() const {
return _gc_state.is_set(MARKING);
}
+inline bool ShenandoahHeap::is_concurrent_young_mark_in_progress() const {
+ return _gc_state.is_set(YOUNG_MARKING);
+}
+
+inline bool ShenandoahHeap::is_concurrent_old_mark_in_progress() const {
+ return _gc_state.is_set(OLD_MARKING);
+}
+
inline bool ShenandoahHeap::is_evacuation_in_progress() const {
return _gc_state.is_set(EVACUATION);
}
@@ -355,8 +511,7 @@ template
inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit) {
assert(! region->is_humongous_continuation(), "no humongous continuation regions here");
- ShenandoahMarkingContext* const ctx = complete_marking_context();
- assert(ctx->is_complete(), "sanity");
+ ShenandoahMarkingContext* const ctx = marking_context();
HeapWord* tams = ctx->top_at_mark_start(region);
@@ -487,14 +642,6 @@ inline ShenandoahHeapRegion* ShenandoahHeap::get_region(size_t region_idx) const
}
}
-inline void ShenandoahHeap::mark_complete_marking_context() {
- _marking_context->mark_complete();
-}
-
-inline void ShenandoahHeap::mark_incomplete_marking_context() {
- _marking_context->mark_incomplete();
-}
-
inline ShenandoahMarkingContext* ShenandoahHeap::complete_marking_context() const {
assert (_marking_context->is_complete()," sanity");
return _marking_context;
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp
index 92602871ccd..9767caf8156 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,12 +25,19 @@
*/
#include "precompiled.hpp"
+#include "gc/shared/cardTable.hpp"
#include "gc/shared/space.hpp"
#include "gc/shared/tlab_globals.hpp"
+#include "gc/shenandoah/shenandoahCardTable.hpp"
+#include "gc/shenandoah/shenandoahFreeSet.hpp"
#include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
+#include "gc/shenandoah/shenandoahGeneration.hpp"
+#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
+#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
#include "jfr/jfrEvents.hpp"
#include "memory/allocation.hpp"
#include "memory/iterator.inline.hpp"
@@ -60,13 +68,20 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool c
_end(start + RegionSizeWords),
_new_top(nullptr),
_empty_time(os::elapsedTime()),
+ _top_before_promoted(nullptr),
_state(committed ? _empty_committed : _empty_uncommitted),
_top(start),
_tlab_allocs(0),
_gclab_allocs(0),
+ _plab_allocs(0),
_live_data(0),
_critical_pins(0),
- _update_watermark(start) {
+ _update_watermark(start),
+ _age(0)
+#ifdef SHENANDOAH_CENSUS_NOISE
+ , _youth(0)
+#endif // SHENANDOAH_CENSUS_NOISE
+ {
assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end),
"invalid space boundaries");
@@ -82,13 +97,14 @@ void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
fatal("%s", ss.freeze());
}
-void ShenandoahHeapRegion::make_regular_allocation() {
+void ShenandoahHeapRegion::make_regular_allocation(ShenandoahAffiliation affiliation) {
shenandoah_assert_heaplocked();
-
+ reset_age();
switch (_state) {
case _empty_uncommitted:
do_commit();
case _empty_committed:
+ assert(this->affiliation() == affiliation, "Region affiliation should already be established");
set_state(_regular);
case _regular:
case _pinned:
@@ -98,13 +114,38 @@ void ShenandoahHeapRegion::make_regular_allocation() {
}
}
+// Change affiliation to YOUNG_GENERATION if _state is not _pinned_cset, _regular, or _pinned. This implements
+// behavior previously performed as a side effect of make_regular_bypass(). This is used by Full GC in non-generational
+// modes to transition regions from FREE. Note that all non-free regions in single-generational modes are young.
+void ShenandoahHeapRegion::make_affiliated_maybe() {
+ shenandoah_assert_heaplocked();
+ assert(!ShenandoahHeap::heap()->mode()->is_generational(), "Only call if non-generational");
+ switch (_state) {
+ case _empty_uncommitted:
+ case _empty_committed:
+ case _cset:
+ case _humongous_start:
+ case _humongous_cont:
+ if (affiliation() != YOUNG_GENERATION) {
+ set_affiliation(YOUNG_GENERATION);
+ }
+ return;
+ case _pinned_cset:
+ case _regular:
+ case _pinned:
+ return;
+ default:
+ assert(false, "Unexpected _state in make_affiliated_maybe");
+ }
+}
+
void ShenandoahHeapRegion::make_regular_bypass() {
shenandoah_assert_heaplocked();
assert (!Universe::is_fully_initialized() ||
ShenandoahHeap::heap()->is_full_gc_in_progress() ||
ShenandoahHeap::heap()->is_degenerated_gc_in_progress(),
"Only for STW GC or when Universe is initializing (CDS)");
-
+ reset_age();
switch (_state) {
case _empty_uncommitted:
do_commit();
@@ -112,6 +153,14 @@ void ShenandoahHeapRegion::make_regular_bypass() {
case _cset:
case _humongous_start:
case _humongous_cont:
+ if (_state == _humongous_start || _state == _humongous_cont) {
+ // CDS allocates chunks of the heap to fill with regular objects. The allocator
+ // will dutifully track any waste in the unused portion of the last region. Once
+ // CDS has finished initializing the objects, it will convert these regions to
+ // regular regions. The 'waste' in the last region is no longer wasted at this point,
+ // so we must stop treating it as such.
+ decrement_humongous_waste();
+ }
set_state(_regular);
return;
case _pinned_cset:
@@ -127,6 +176,7 @@ void ShenandoahHeapRegion::make_regular_bypass() {
void ShenandoahHeapRegion::make_humongous_start() {
shenandoah_assert_heaplocked();
+ reset_age();
switch (_state) {
case _empty_uncommitted:
do_commit();
@@ -138,10 +188,12 @@ void ShenandoahHeapRegion::make_humongous_start() {
}
}
-void ShenandoahHeapRegion::make_humongous_start_bypass() {
+void ShenandoahHeapRegion::make_humongous_start_bypass(ShenandoahAffiliation affiliation) {
shenandoah_assert_heaplocked();
assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
-
+ // Don't bother to account for affiliated regions during Full GC. We recompute totals at end.
+ set_affiliation(affiliation);
+ reset_age();
switch (_state) {
case _empty_committed:
case _regular:
@@ -156,6 +208,7 @@ void ShenandoahHeapRegion::make_humongous_start_bypass() {
void ShenandoahHeapRegion::make_humongous_cont() {
shenandoah_assert_heaplocked();
+ reset_age();
switch (_state) {
case _empty_uncommitted:
do_commit();
@@ -167,10 +220,12 @@ void ShenandoahHeapRegion::make_humongous_cont() {
}
}
-void ShenandoahHeapRegion::make_humongous_cont_bypass() {
+void ShenandoahHeapRegion::make_humongous_cont_bypass(ShenandoahAffiliation affiliation) {
shenandoah_assert_heaplocked();
assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
-
+ set_affiliation(affiliation);
+ // Don't bother to account for affiliated regions during Full GC. We recompute totals at end.
+ reset_age();
switch (_state) {
case _empty_committed:
case _regular:
@@ -211,6 +266,7 @@ void ShenandoahHeapRegion::make_unpinned() {
switch (_state) {
case _pinned:
+ assert(is_affiliated(), "Pinned region should be affiliated");
set_state(_regular);
return;
case _regular:
@@ -229,6 +285,7 @@ void ShenandoahHeapRegion::make_unpinned() {
void ShenandoahHeapRegion::make_cset() {
shenandoah_assert_heaplocked();
+ // Leave age untouched. We need to consult the age when we are deciding whether to promote evacuated objects.
switch (_state) {
case _regular:
set_state(_cset);
@@ -241,12 +298,17 @@ void ShenandoahHeapRegion::make_cset() {
void ShenandoahHeapRegion::make_trash() {
shenandoah_assert_heaplocked();
+ reset_age();
switch (_state) {
- case _cset:
- // Reclaiming cset regions
case _humongous_start:
case _humongous_cont:
- // Reclaiming humongous regions
+ {
+ // Reclaiming humongous regions and reclaim humongous waste. When this region is eventually recycled, we'll reclaim
+ // its used memory. At recycle time, we no longer recognize this as a humongous region.
+ decrement_humongous_waste();
+ }
+ case _cset:
+ // Reclaiming cset regions
case _regular:
// Immediate region reclaim
set_state(_trash);
@@ -261,11 +323,15 @@ void ShenandoahHeapRegion::make_trash_immediate() {
// On this path, we know there are no marked objects in the region,
// tell marking context about it to bypass bitmap resets.
- ShenandoahHeap::heap()->complete_marking_context()->reset_top_bitmap(this);
+ assert(ShenandoahHeap::heap()->gc_generation()->is_mark_complete(), "Marking should be complete here.");
+ shenandoah_assert_generations_reconciled();
+ ShenandoahHeap::heap()->marking_context()->reset_top_bitmap(this);
}
void ShenandoahHeapRegion::make_empty() {
shenandoah_assert_heaplocked();
+ reset_age();
+ CENSUS_NOISE(clear_youth();)
switch (_state) {
case _trash:
set_state(_empty_committed);
@@ -305,10 +371,11 @@ void ShenandoahHeapRegion::make_committed_bypass() {
void ShenandoahHeapRegion::reset_alloc_metadata() {
_tlab_allocs = 0;
_gclab_allocs = 0;
+ _plab_allocs = 0;
}
size_t ShenandoahHeapRegion::get_shared_allocs() const {
- return used() - (_tlab_allocs + _gclab_allocs) * HeapWordSize;
+ return used() - (_tlab_allocs + _gclab_allocs + _plab_allocs) * HeapWordSize;
}
size_t ShenandoahHeapRegion::get_tlab_allocs() const {
@@ -319,6 +386,10 @@ size_t ShenandoahHeapRegion::get_gclab_allocs() const {
return _gclab_allocs * HeapWordSize;
}
+size_t ShenandoahHeapRegion::get_plab_allocs() const {
+ return _plab_allocs * HeapWordSize;
+}
+
void ShenandoahHeapRegion::set_live_data(size_t s) {
assert(Thread::current()->is_VM_thread(), "by VM thread");
_live_data = (s >> LogHeapWordSize);
@@ -363,6 +434,8 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const {
ShouldNotReachHere();
}
+ st->print("|%s", shenandoah_affiliation_code(affiliation()));
+
#define SHR_PTR_FORMAT "%12" PRIxPTR
st->print("|BTE " SHR_PTR_FORMAT ", " SHR_PTR_FORMAT ", " SHR_PTR_FORMAT,
@@ -374,6 +447,9 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const {
st->print("|U " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used()));
st->print("|T " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_tlab_allocs()), proper_unit_for_byte_size(get_tlab_allocs()));
st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()), proper_unit_for_byte_size(get_gclab_allocs()));
+ if (ShenandoahHeap::heap()->mode()->is_generational()) {
+ st->print("|P " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_plab_allocs()), proper_unit_for_byte_size(get_plab_allocs()));
+ }
st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()), proper_unit_for_byte_size(get_shared_allocs()));
st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));
st->print("|CP " SIZE_FORMAT_W(3), pin_count());
@@ -382,33 +458,100 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const {
#undef SHR_PTR_FORMAT
}
-void ShenandoahHeapRegion::oop_iterate(OopIterateClosure* blk) {
- if (!is_active()) return;
- if (is_humongous()) {
- oop_iterate_humongous(blk);
- } else {
- oop_iterate_objects(blk);
+// oop_iterate without closure, return true if completed without cancellation
+bool ShenandoahHeapRegion::oop_coalesce_and_fill(bool cancellable) {
+
+ assert(!is_humongous(), "No need to fill or coalesce humongous regions");
+ if (!is_active()) {
+ end_preemptible_coalesce_and_fill();
+ return true;
}
-}
-void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk) {
- assert(! is_humongous(), "no humongous region here");
- HeapWord* obj_addr = bottom();
- HeapWord* t = top();
- // Could call objects iterate, but this is easier.
+ ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
+ ShenandoahMarkingContext* marking_context = heap->marking_context();
+
+ // Expect marking to be completed before these threads invoke this service.
+ assert(heap->gc_generation()->is_mark_complete(), "sanity");
+ shenandoah_assert_generations_reconciled();
+
+ // All objects above TAMS are considered live even though their mark bits will not be set. Note that young-
+ // gen evacuations that interrupt a long-running old-gen concurrent mark may promote objects into old-gen
+ // while the old-gen concurrent marking is ongoing. These newly promoted objects will reside above TAMS
+ // and will be treated as live during the current old-gen marking pass, even though they will not be
+ // explicitly marked.
+ HeapWord* t = marking_context->top_at_mark_start(this);
+
+ // Resume coalesce and fill from this address
+ HeapWord* obj_addr = resume_coalesce_and_fill();
+
while (obj_addr < t) {
oop obj = cast_to_oop(obj_addr);
- obj_addr += obj->oop_iterate_size(blk);
+ if (marking_context->is_marked(obj)) {
+ assert(obj->klass() != nullptr, "klass should not be nullptr");
+ obj_addr += obj->size();
+ } else {
+ // Object is not marked. Coalesce and fill dead object with dead neighbors.
+ HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t);
+ assert(next_marked_obj <= t, "next marked object cannot exceed top");
+ size_t fill_size = next_marked_obj - obj_addr;
+ assert(fill_size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size");
+ ShenandoahHeap::fill_with_object(obj_addr, fill_size);
+ heap->old_generation()->card_scan()->coalesce_objects(obj_addr, fill_size);
+ obj_addr = next_marked_obj;
+ }
+ if (cancellable && heap->cancelled_gc()) {
+ suspend_coalesce_and_fill(obj_addr);
+ return false;
+ }
}
+ // Mark that this region has been coalesced and filled
+ end_preemptible_coalesce_and_fill();
+ return true;
}
-void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) {
+size_t get_card_count(size_t words) {
+ assert(words % CardTable::card_size_in_words() == 0, "Humongous iteration must span whole number of cards");
+ assert(CardTable::card_size_in_words() * (words / CardTable::card_size_in_words()) == words,
+ "slice must be integral number of cards");
+ return words / CardTable::card_size_in_words();
+}
+
+void ShenandoahHeapRegion::oop_iterate_humongous_slice_dirty(OopIterateClosure* blk,
+ HeapWord* start, size_t words, bool write_table) const {
assert(is_humongous(), "only humongous region here");
- // Find head.
+
ShenandoahHeapRegion* r = humongous_start_region();
- assert(r->is_humongous_start(), "need humongous head here");
oop obj = cast_to_oop(r->bottom());
- obj->oop_iterate(blk, MemRegion(bottom(), top()));
+ size_t num_cards = get_card_count(words);
+
+ ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
+ ShenandoahScanRemembered* scanner = heap->old_generation()->card_scan();
+ size_t card_index = scanner->card_index_for_addr(start);
+ if (write_table) {
+ while (num_cards-- > 0) {
+ if (scanner->is_write_card_dirty(card_index++)) {
+ obj->oop_iterate(blk, MemRegion(start, start + CardTable::card_size_in_words()));
+ }
+ start += CardTable::card_size_in_words();
+ }
+ } else {
+ while (num_cards-- > 0) {
+ if (scanner->is_card_dirty(card_index++)) {
+ obj->oop_iterate(blk, MemRegion(start, start + CardTable::card_size_in_words()));
+ }
+ start += CardTable::card_size_in_words();
+ }
+ }
+}
+
+void ShenandoahHeapRegion::oop_iterate_humongous_slice_all(OopIterateClosure* cl, HeapWord* start, size_t words) const {
+ assert(is_humongous(), "only humongous region here");
+
+ ShenandoahHeapRegion* r = humongous_start_region();
+ oop obj = cast_to_oop(r->bottom());
+
+ // Scan all data, regardless of whether cards are dirty
+ obj->oop_iterate(cl, MemRegion(start, start + words));
}
ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
@@ -427,16 +570,24 @@ ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
}
void ShenandoahHeapRegion::recycle() {
+ shenandoah_assert_heaplocked();
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+ ShenandoahGeneration* generation = heap->generation_for(affiliation());
+
+ heap->decrease_used(generation, used());
+ generation->decrement_affiliated_region_count();
+
set_top(bottom());
clear_live_data();
-
reset_alloc_metadata();
- ShenandoahHeap::heap()->marking_context()->reset_top_at_mark_start(this);
+ heap->marking_context()->reset_top_at_mark_start(this);
+
set_update_watermark(bottom());
make_empty();
+ set_affiliation(FREE);
if (ZapUnusedHeapArea) {
SpaceMangler::mangle_region(MemRegion(bottom(), end()));
}
@@ -480,6 +631,11 @@ size_t ShenandoahHeapRegion::setup_sizes(size_t max_heap_size) {
FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE);
}
+ // Generational Shenandoah needs this alignment for card tables.
+ if (strcmp(ShenandoahGCMode, "generational") == 0) {
+ max_heap_size = align_up(max_heap_size , CardTable::ct_max_alignment_constraint());
+ }
+
size_t region_size;
if (FLAG_IS_DEFAULT(ShenandoahRegionSize)) {
if (ShenandoahMinRegionSize > max_heap_size / MIN_NUM_REGIONS) {
@@ -658,3 +814,65 @@ void ShenandoahHeapRegion::record_unpin() {
size_t ShenandoahHeapRegion::pin_count() const {
return Atomic::load(&_critical_pins);
}
+
+void ShenandoahHeapRegion::set_affiliation(ShenandoahAffiliation new_affiliation) {
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+
+ ShenandoahAffiliation region_affiliation = heap->region_affiliation(this);
+ {
+ ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
+ log_debug(gc)("Setting affiliation of Region " SIZE_FORMAT " from %s to %s, top: " PTR_FORMAT ", TAMS: " PTR_FORMAT
+ ", watermark: " PTR_FORMAT ", top_bitmap: " PTR_FORMAT,
+ index(), shenandoah_affiliation_name(region_affiliation), shenandoah_affiliation_name(new_affiliation),
+ p2i(top()), p2i(ctx->top_at_mark_start(this)), p2i(_update_watermark), p2i(ctx->top_bitmap(this)));
+ }
+
+#ifdef ASSERT
+ {
+ // During full gc, heap->complete_marking_context() is not valid, may equal nullptr.
+ ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
+ size_t idx = this->index();
+ HeapWord* top_bitmap = ctx->top_bitmap(this);
+
+ assert(ctx->is_bitmap_range_within_region_clear(top_bitmap, _end),
+ "Region " SIZE_FORMAT ", bitmap should be clear between top_bitmap: " PTR_FORMAT " and end: " PTR_FORMAT, idx,
+ p2i(top_bitmap), p2i(_end));
+ }
+#endif
+
+ if (region_affiliation == new_affiliation) {
+ return;
+ }
+
+ if (!heap->mode()->is_generational()) {
+ log_trace(gc)("Changing affiliation of region %zu from %s to %s",
+ index(), affiliation_name(), shenandoah_affiliation_name(new_affiliation));
+ heap->set_affiliation(this, new_affiliation);
+ return;
+ }
+
+ switch (new_affiliation) {
+ case FREE:
+ assert(!has_live(), "Free region should not have live data");
+ break;
+ case YOUNG_GENERATION:
+ reset_age();
+ break;
+ case OLD_GENERATION:
+ break;
+ default:
+ ShouldNotReachHere();
+ return;
+ }
+ heap->set_affiliation(this, new_affiliation);
+}
+
+void ShenandoahHeapRegion::decrement_humongous_waste() const {
+ assert(is_humongous(), "Should only use this for humongous regions");
+ size_t waste_bytes = free();
+ if (waste_bytes > 0) {
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+ ShenandoahGeneration* generation = heap->generation_for(affiliation());
+ heap->decrease_humongous_waste(generation, waste_bytes);
+ }
+}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp
index c34c4c232e4..1c0f1182f07 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -27,6 +28,8 @@
#include "gc/shared/gc_globals.hpp"
#include "gc/shared/spaceDecorator.hpp"
+#include "gc/shenandoah/shenandoahAffiliation.hpp"
+#include "gc/shenandoah/shenandoahAgeCensus.hpp"
#include "gc/shenandoah/shenandoahAllocRequest.hpp"
#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
@@ -122,6 +125,7 @@ class ShenandoahHeapRegion {
_REGION_STATES_NUM // last
};
+public:
static const char* region_state_to_string(RegionState s) {
switch (s) {
case _empty_uncommitted: return "Empty Uncommitted";
@@ -140,6 +144,7 @@ class ShenandoahHeapRegion {
}
}
+private:
// This method protects from accidental changes in enum order:
int region_state_to_ordinal(RegionState s) const {
switch (s) {
@@ -167,12 +172,13 @@ class ShenandoahHeapRegion {
}
// Allowed transitions from the outside code:
- void make_regular_allocation();
+ void make_regular_allocation(ShenandoahAffiliation affiliation);
+ void make_affiliated_maybe();
void make_regular_bypass();
void make_humongous_start();
void make_humongous_cont();
- void make_humongous_start_bypass();
- void make_humongous_cont_bypass();
+ void make_humongous_start_bypass(ShenandoahAffiliation affiliation);
+ void make_humongous_cont_bypass(ShenandoahAffiliation affiliation);
void make_pinned();
void make_unpinned();
void make_cset();
@@ -197,6 +203,11 @@ class ShenandoahHeapRegion {
bool is_committed() const { return !is_empty_uncommitted(); }
bool is_cset() const { return _state == _cset || _state == _pinned_cset; }
bool is_pinned() const { return _state == _pinned || _state == _pinned_cset || _state == _pinned_humongous_start; }
+ bool is_regular_pinned() const { return _state == _pinned; }
+
+ inline bool is_young() const;
+ inline bool is_old() const;
+ inline bool is_affiliated() const;
// Macro-properties:
bool is_alloc_allowed() const { return is_empty() || is_regular() || _state == _pinned; }
@@ -229,20 +240,27 @@ class ShenandoahHeapRegion {
HeapWord* _new_top;
double _empty_time;
+ HeapWord* _top_before_promoted;
+
// Seldom updated fields
RegionState _state;
+ HeapWord* _coalesce_and_fill_boundary; // for old regions not selected as collection set candidates.
// Frequently updated fields
HeapWord* _top;
size_t _tlab_allocs;
size_t _gclab_allocs;
+ size_t _plab_allocs;
volatile size_t _live_data;
volatile size_t _critical_pins;
HeapWord* volatile _update_watermark;
+ uint _age;
+ CENSUS_NOISE(uint _youth;) // tracks epochs of retrograde ageing (rejuvenation)
+
public:
ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed);
@@ -327,8 +345,19 @@ class ShenandoahHeapRegion {
return _index;
}
- // Allocation (return null if full)
- inline HeapWord* allocate(size_t word_size, ShenandoahAllocRequest::Type type);
+ inline void save_top_before_promote();
+ inline HeapWord* get_top_before_promote() const { return _top_before_promoted; }
+ inline void restore_top_before_promote();
+ inline size_t garbage_before_padded_for_promote() const;
+
+ // If next available memory is not aligned on address that is multiple of alignment, fill the empty space
+ // so that returned object is aligned on an address that is a multiple of alignment_in_bytes. Requested
+ // size is in words. It is assumed that this->is_old(). A pad object is allocated, filled, and registered
+ // if necessary to assure the new allocation is properly aligned. Return nullptr if memory is not available.
+ inline HeapWord* allocate_aligned(size_t word_size, ShenandoahAllocRequest &req, size_t alignment_in_bytes);
+
+ // Allocation (return nullptr if full)
+ inline HeapWord* allocate(size_t word_size, const ShenandoahAllocRequest& req);
inline void clear_live_data();
void set_live_data(size_t s);
@@ -349,7 +378,36 @@ class ShenandoahHeapRegion {
void recycle();
- void oop_iterate(OopIterateClosure* cl);
+ inline void begin_preemptible_coalesce_and_fill() {
+ _coalesce_and_fill_boundary = _bottom;
+ }
+
+ inline void end_preemptible_coalesce_and_fill() {
+ _coalesce_and_fill_boundary = _end;
+ }
+
+ inline void suspend_coalesce_and_fill(HeapWord* next_focus) {
+ _coalesce_and_fill_boundary = next_focus;
+ }
+
+ inline HeapWord* resume_coalesce_and_fill() {
+ return _coalesce_and_fill_boundary;
+ }
+
+ // Coalesce contiguous spans of garbage objects by filling header and registering start locations with remembered set.
+ // This is used by old-gen GC following concurrent marking to make old-gen HeapRegions parsable. Old regions must be
+ // parsable because the mark bitmap is not reliable during the concurrent old mark.
+ // Return true iff region is completely coalesced and filled. Returns false if cancelled before task is complete.
+ bool oop_coalesce_and_fill(bool cancellable);
+
+ // Invoke closure on every reference contained within the humongous object that spans this humongous
+ // region if the reference is contained within a DIRTY card and the reference is no more than words following
+ // start within the humongous object.
+ void oop_iterate_humongous_slice_dirty(OopIterateClosure* cl, HeapWord* start, size_t words, bool write_table) const;
+
+ // Invoke closure on every reference contained within the humongous object starting from start and
+ // ending at start + words.
+ void oop_iterate_humongous_slice_all(OopIterateClosure* cl, HeapWord* start, size_t words) const;
HeapWord* block_start(const void* p) const;
size_t block_size(const HeapWord* p) const;
@@ -369,25 +427,54 @@ class ShenandoahHeapRegion {
size_t capacity() const { return byte_size(bottom(), end()); }
size_t used() const { return byte_size(bottom(), top()); }
+ size_t used_before_promote() const { return byte_size(bottom(), get_top_before_promote()); }
size_t free() const { return byte_size(top(), end()); }
+ // Does this region contain this address?
+ bool contains(HeapWord* p) const {
+ return (bottom() <= p) && (p < top());
+ }
+
inline void adjust_alloc_metadata(ShenandoahAllocRequest::Type type, size_t);
void reset_alloc_metadata();
size_t get_shared_allocs() const;
size_t get_tlab_allocs() const;
size_t get_gclab_allocs() const;
+ size_t get_plab_allocs() const;
inline HeapWord* get_update_watermark() const;
inline void set_update_watermark(HeapWord* w);
inline void set_update_watermark_at_safepoint(HeapWord* w);
+ inline ShenandoahAffiliation affiliation() const;
+ inline const char* affiliation_name() const;
+
+ void set_affiliation(ShenandoahAffiliation new_affiliation);
+
+ // Region ageing and rejuvenation
+ uint age() const { return _age; }
+ CENSUS_NOISE(uint youth() const { return _youth; })
+
+ void increment_age() {
+ const uint max_age = markWord::max_age;
+ assert(_age <= max_age, "Error");
+ if (_age++ >= max_age) {
+ _age = max_age; // clamp
+ }
+ }
+
+ void reset_age() {
+ CENSUS_NOISE(_youth += _age;)
+ _age = 0;
+ }
+
+ CENSUS_NOISE(void clear_youth() { _youth = 0; })
+
private:
+ void decrement_humongous_waste() const;
void do_commit();
void do_uncommit();
- void oop_iterate_objects(OopIterateClosure* cl);
- void oop_iterate_humongous(OopIterateClosure* cl);
-
inline void internal_increase_live_data(size_t s);
void set_state(RegionState to);
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp
index 0435333fe1e..382d9ba942c 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2015, 2019, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,20 +26,74 @@
#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_INLINE_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_INLINE_HPP
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
-
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
#include "gc/shenandoah/shenandoahPacer.inline.hpp"
#include "runtime/atomic.hpp"
-HeapWord* ShenandoahHeapRegion::allocate(size_t size, ShenandoahAllocRequest::Type type) {
+HeapWord* ShenandoahHeapRegion::allocate_aligned(size_t size, ShenandoahAllocRequest &req, size_t alignment_in_bytes) {
+ shenandoah_assert_heaplocked_or_safepoint();
+ assert(req.is_lab_alloc(), "allocate_aligned() only applies to LAB allocations");
+ assert(is_object_aligned(size), "alloc size breaks alignment: " SIZE_FORMAT, size);
+ assert(is_old(), "aligned allocations are only taken from OLD regions to support PLABs");
+ assert(is_aligned(alignment_in_bytes, HeapWordSize), "Expect heap word alignment");
+
+ HeapWord* orig_top = top();
+ size_t alignment_in_words = alignment_in_bytes / HeapWordSize;
+
+ // unalignment_words is the amount by which current top() exceeds the desired alignment point. We subtract this amount
+ // from alignment_in_words to determine padding required to next alignment point.
+
+ HeapWord* aligned_obj = (HeapWord*) align_up(orig_top, alignment_in_bytes);
+ size_t pad_words = aligned_obj - orig_top;
+ if ((pad_words > 0) && (pad_words < ShenandoahHeap::min_fill_size())) {
+ pad_words += alignment_in_words;
+ aligned_obj += alignment_in_words;
+ }
+
+ if (pointer_delta(end(), aligned_obj) < size) {
+ // Shrink size to fit within available space and align it
+ size = pointer_delta(end(), aligned_obj);
+ size = align_down(size, alignment_in_words);
+ }
+
+ // Both originally requested size and adjusted size must be properly aligned
+ assert (is_aligned(size, alignment_in_words), "Size must be multiple of alignment constraint");
+ if (size >= req.min_size()) {
+ // Even if req.min_size() may not be a multiple of card size, we know that size is.
+ if (pad_words > 0) {
+ assert(pad_words >= ShenandoahHeap::min_fill_size(), "pad_words expanded above to meet size constraint");
+ ShenandoahHeap::fill_with_object(orig_top, pad_words);
+ ShenandoahGenerationalHeap::heap()->old_generation()->card_scan()->register_object(orig_top);
+ }
+
+ make_regular_allocation(req.affiliation());
+ adjust_alloc_metadata(req.type(), size);
+
+ HeapWord* new_top = aligned_obj + size;
+ assert(new_top <= end(), "PLAB cannot span end of heap region");
+ set_top(new_top);
+ // We do not req.set_actual_size() here. The caller sets it.
+ req.set_waste(pad_words);
+ assert(is_object_aligned(new_top), "new top breaks alignment: " PTR_FORMAT, p2i(new_top));
+ assert(is_aligned(aligned_obj, alignment_in_bytes), "obj is not aligned: " PTR_FORMAT, p2i(aligned_obj));
+ return aligned_obj;
+ } else {
+ // The aligned size that fits in this region is smaller than min_size, so don't align top and don't allocate. Return failure.
+ return nullptr;
+ }
+}
+
+HeapWord* ShenandoahHeapRegion::allocate(size_t size, const ShenandoahAllocRequest& req) {
shenandoah_assert_heaplocked_or_safepoint();
assert(is_object_aligned(size), "alloc size breaks alignment: " SIZE_FORMAT, size);
HeapWord* obj = top();
if (pointer_delta(end(), obj) >= size) {
- make_regular_allocation();
- adjust_alloc_metadata(type, size);
+ make_regular_allocation(req.affiliation());
+ adjust_alloc_metadata(req.type(), size);
HeapWord* new_top = obj + size;
set_top(new_top);
@@ -64,6 +119,9 @@ inline void ShenandoahHeapRegion::adjust_alloc_metadata(ShenandoahAllocRequest::
case ShenandoahAllocRequest::_alloc_gclab:
_gclab_allocs += size;
break;
+ case ShenandoahAllocRequest::_alloc_plab:
+ _plab_allocs += size;
+ break;
default:
ShouldNotReachHere();
}
@@ -82,12 +140,6 @@ inline void ShenandoahHeapRegion::increase_live_data_gc_words(size_t s) {
inline void ShenandoahHeapRegion::internal_increase_live_data(size_t s) {
size_t new_live_data = Atomic::add(&_live_data, s, memory_order_relaxed);
-#ifdef ASSERT
- size_t live_bytes = new_live_data * HeapWordSize;
- size_t used_bytes = used();
- assert(live_bytes <= used_bytes,
- "can't have more live data than used: " SIZE_FORMAT ", " SIZE_FORMAT, live_bytes, used_bytes);
-#endif
}
inline void ShenandoahHeapRegion::clear_live_data() {
@@ -115,6 +167,17 @@ inline size_t ShenandoahHeapRegion::garbage() const {
return result;
}
+inline size_t ShenandoahHeapRegion::garbage_before_padded_for_promote() const {
+ assert(get_top_before_promote() != nullptr, "top before promote should not equal null");
+ size_t used_before_promote = byte_size(bottom(), get_top_before_promote());
+ assert(used_before_promote >= get_live_data_bytes(),
+ "Live Data must be a subset of used before promotion live: " SIZE_FORMAT " used: " SIZE_FORMAT,
+ get_live_data_bytes(), used_before_promote);
+ size_t result = used_before_promote - get_live_data_bytes();
+ return result;
+
+}
+
inline HeapWord* ShenandoahHeapRegion::get_update_watermark() const {
HeapWord* watermark = Atomic::load_acquire(&_update_watermark);
assert(bottom() <= watermark && watermark <= top(), "within bounds");
@@ -133,4 +196,34 @@ inline void ShenandoahHeapRegion::set_update_watermark_at_safepoint(HeapWord* w)
_update_watermark = w;
}
+inline ShenandoahAffiliation ShenandoahHeapRegion::affiliation() const {
+ return ShenandoahHeap::heap()->region_affiliation(this);
+}
+
+inline const char* ShenandoahHeapRegion::affiliation_name() const {
+ return shenandoah_affiliation_name(affiliation());
+}
+
+inline bool ShenandoahHeapRegion::is_young() const {
+ return affiliation() == YOUNG_GENERATION;
+}
+
+inline bool ShenandoahHeapRegion::is_old() const {
+ return affiliation() == OLD_GENERATION;
+}
+
+inline bool ShenandoahHeapRegion::is_affiliated() const {
+ return affiliation() != FREE;
+}
+
+inline void ShenandoahHeapRegion::save_top_before_promote() {
+ _top_before_promoted = _top;
+}
+
+inline void ShenandoahHeapRegion::restore_top_before_promote() {
+ _top = _top_before_promoted;
+ _top_before_promoted = nullptr;
+ }
+
+
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_INLINE_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionClosures.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionClosures.cpp
new file mode 100644
index 00000000000..3d3483a5b69
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionClosures.cpp
@@ -0,0 +1,89 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
+#include "gc/shenandoah/shenandoahMarkingContext.hpp"
+#include "gc/shenandoah/shenandoahSharedVariables.hpp"
+
+ShenandoahSynchronizePinnedRegionStates::ShenandoahSynchronizePinnedRegionStates() :
+ _lock(ShenandoahHeap::heap()->lock()) { }
+
+void ShenandoahSynchronizePinnedRegionStates::heap_region_do(ShenandoahHeapRegion* r) {
+ // Drop "pinned" state from regions that no longer have a pinned count. Put
+ // regions with a pinned count into the "pinned" state.
+ if (r->is_active()) {
+ synchronize_pin_count(r);
+ }
+}
+
+void ShenandoahSynchronizePinnedRegionStates::synchronize_pin_count(ShenandoahHeapRegion* r) {
+ if (r->is_pinned()) {
+ if (r->pin_count() == 0) {
+ ShenandoahHeapLocker locker(_lock);
+ r->make_unpinned();
+ }
+ } else {
+ if (r->pin_count() > 0) {
+ ShenandoahHeapLocker locker(_lock);
+ r->make_pinned();
+ }
+ }
+}
+
+ShenandoahFinalMarkUpdateRegionStateClosure::ShenandoahFinalMarkUpdateRegionStateClosure(ShenandoahMarkingContext *ctx) :
+ _ctx(ctx) { }
+
+void ShenandoahFinalMarkUpdateRegionStateClosure::heap_region_do(ShenandoahHeapRegion* r) {
+ if (r->is_active()) {
+ if (_ctx != nullptr) {
+ // _ctx may be null when this closure is used to sync only the pin status
+ // update the watermark of old regions. For old regions we cannot reset
+ // the TAMS because we rely on that to keep promoted objects alive after
+ // old marking is complete.
+
+ // All allocations past TAMS are implicitly live, adjust the region data.
+ // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
+ HeapWord *tams = _ctx->top_at_mark_start(r);
+ HeapWord *top = r->top();
+ if (top > tams) {
+ r->increase_live_data_alloc_words(pointer_delta(top, tams));
+ }
+ }
+
+ // We are about to select the collection set, make sure it knows about
+ // current pinning status. Also, this allows trashing more regions that
+ // now have their pinning status dropped.
+ _pins.synchronize_pin_count(r);
+
+ // Remember limit for updating refs. It's guaranteed that we get no
+ // from-space-refs written from here on.
+ r->set_update_watermark_at_safepoint(r->top());
+ } else {
+ assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
+ assert(_ctx == nullptr || _ctx->top_at_mark_start(r) == r->top(),
+ "Region " SIZE_FORMAT " should have correct TAMS", r->index());
+ }
+}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionClosures.hpp
new file mode 100644
index 00000000000..0daf268628c
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionClosures.hpp
@@ -0,0 +1,100 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGIONCLOSURES_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGIONCLOSURES_HPP
+
+
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
+
+// Applies the given closure to all regions with the given affiliation
+template
+class ShenandoahIncludeRegionClosure : public ShenandoahHeapRegionClosure {
+private:
+ ShenandoahHeapRegionClosure* _closure;
+
+public:
+ explicit ShenandoahIncludeRegionClosure(ShenandoahHeapRegionClosure* closure): _closure(closure) {}
+
+ void heap_region_do(ShenandoahHeapRegion* r) override {
+ if (r->affiliation() == AFFILIATION) {
+ _closure->heap_region_do(r);
+ }
+ }
+
+ bool is_thread_safe() override {
+ return _closure->is_thread_safe();
+ }
+};
+
+// Applies the given closure to all regions without the given affiliation
+template
+class ShenandoahExcludeRegionClosure : public ShenandoahHeapRegionClosure {
+private:
+ ShenandoahHeapRegionClosure* _closure;
+
+public:
+ explicit ShenandoahExcludeRegionClosure(ShenandoahHeapRegionClosure* closure): _closure(closure) {}
+
+ void heap_region_do(ShenandoahHeapRegion* r) override {
+ if (r->affiliation() != AFFILIATION) {
+ _closure->heap_region_do(r);
+ }
+ }
+
+ bool is_thread_safe() override {
+ return _closure->is_thread_safe();
+ }
+};
+
+// Makes regions pinned or unpinned according to the region's pin count
+class ShenandoahSynchronizePinnedRegionStates : public ShenandoahHeapRegionClosure {
+private:
+ ShenandoahHeapLock* const _lock;
+
+public:
+ ShenandoahSynchronizePinnedRegionStates();
+
+ void heap_region_do(ShenandoahHeapRegion* r) override;
+ bool is_thread_safe() override { return true; }
+
+ void synchronize_pin_count(ShenandoahHeapRegion* r);
+};
+
+class ShenandoahMarkingContext;
+
+// Synchronizes region pinned status, sets update watermark and adjust live data tally for regions
+class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
+private:
+ ShenandoahMarkingContext* const _ctx;
+ ShenandoahSynchronizePinnedRegionStates _pins;
+public:
+ explicit ShenandoahFinalMarkUpdateRegionStateClosure(ShenandoahMarkingContext* ctx);
+
+ void heap_region_do(ShenandoahHeapRegion* r) override;
+ bool is_thread_safe() override { return true; }
+};
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGIONCLOSURES_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp
index 3fb6b329f2c..73250cecd6f 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2016, 2020, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -23,13 +24,17 @@
*/
#include "precompiled.hpp"
+
+#include "gc/shenandoah/shenandoahGeneration.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
#include "gc/shenandoah/shenandoahHeapRegionCounters.hpp"
+#include "logging/logStream.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/atomic.hpp"
#include "runtime/perfData.inline.hpp"
+#include "utilities/defaultStream.hpp"
ShenandoahHeapRegionCounters::ShenandoahHeapRegionCounters() :
_last_sample_millis(0)
@@ -49,6 +54,9 @@ ShenandoahHeapRegionCounters::ShenandoahHeapRegionCounters() :
cname = PerfDataManager::counter_name(_name_space, "max_regions");
PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, num_regions, CHECK);
+ cname = PerfDataManager::counter_name(_name_space, "protocol_version");
+ PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, VERSION_NUMBER, CHECK);
+
cname = PerfDataManager::counter_name(_name_space, "region_size");
PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, ShenandoahHeapRegion::region_size_bytes() >> 10, CHECK);
@@ -57,14 +65,14 @@ ShenandoahHeapRegionCounters::ShenandoahHeapRegionCounters() :
PerfData::U_None, CHECK);
_regions_data = NEW_C_HEAP_ARRAY(PerfVariable*, num_regions, mtGC);
+ // Initializing performance data resources for each region
for (uint i = 0; i < num_regions; i++) {
const char* reg_name = PerfDataManager::name_space(_name_space, "region", i);
const char* data_name = PerfDataManager::counter_name(reg_name, "data");
const char* ns = PerfDataManager::ns_to_string(SUN_GC);
const char* fullname = PerfDataManager::counter_name(ns, data_name);
assert(!PerfDataManager::exists(fullname), "must not exist");
- _regions_data[i] = PerfDataManager::create_long_variable(SUN_GC, data_name,
- PerfData::U_None, CHECK);
+ _regions_data[i] = PerfDataManager::create_long_variable(SUN_GC, data_name, PerfData::U_None, CHECK);
}
}
}
@@ -73,27 +81,42 @@ ShenandoahHeapRegionCounters::~ShenandoahHeapRegionCounters() {
if (_name_space != nullptr) FREE_C_HEAP_ARRAY(char, _name_space);
}
+void ShenandoahHeapRegionCounters::write_snapshot(PerfLongVariable** regions,
+ PerfLongVariable* ts,
+ PerfLongVariable* status,
+ size_t num_regions,
+ size_t region_size, size_t protocol_version) {
+ LogTarget(Trace, gc, region) lt;
+ if (lt.is_enabled()) {
+ ResourceMark rm;
+ LogStream ls(lt);
+
+ ls.print_cr(JLONG_FORMAT " " JLONG_FORMAT " " SIZE_FORMAT " " SIZE_FORMAT " " SIZE_FORMAT,
+ ts->get_value(), status->get_value(), num_regions, region_size, protocol_version);
+ if (num_regions > 0) {
+ ls.print(JLONG_FORMAT, regions[0]->get_value());
+ }
+ for (uint i = 1; i < num_regions; ++i) {
+ ls.print(" " JLONG_FORMAT, regions[i]->get_value());
+ }
+ ls.cr();
+ }
+}
+
void ShenandoahHeapRegionCounters::update() {
if (ShenandoahRegionSampling) {
jlong current = nanos_to_millis(os::javaTimeNanos());
jlong last = _last_sample_millis;
- if (current - last > ShenandoahRegionSamplingRate &&
- Atomic::cmpxchg(&_last_sample_millis, last, current) == last) {
+ if (current - last > ShenandoahRegionSamplingRate && Atomic::cmpxchg(&_last_sample_millis, last, current) == last) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
- jlong status = 0;
- if (heap->is_concurrent_mark_in_progress()) status |= 1 << 0;
- if (heap->is_evacuation_in_progress()) status |= 1 << 1;
- if (heap->is_update_refs_in_progress()) status |= 1 << 2;
- _status->set_value(status);
-
+ _status->set_value(encode_heap_status(heap));
_timestamp->set_value(os::elapsed_counter());
- size_t num_regions = heap->num_regions();
-
{
ShenandoahHeapLocker locker(heap->lock());
size_t rs = ShenandoahHeapRegion::region_size_bytes();
+ size_t num_regions = heap->num_regions();
for (uint i = 0; i < num_regions; i++) {
ShenandoahHeapRegion* r = heap->get_region(i);
jlong data = 0;
@@ -101,12 +124,79 @@ void ShenandoahHeapRegionCounters::update() {
data |= ((100 * r->get_live_data_bytes() / rs) & PERCENT_MASK) << LIVE_SHIFT;
data |= ((100 * r->get_tlab_allocs() / rs) & PERCENT_MASK) << TLAB_SHIFT;
data |= ((100 * r->get_gclab_allocs() / rs) & PERCENT_MASK) << GCLAB_SHIFT;
+ data |= ((100 * r->get_plab_allocs() / rs) & PERCENT_MASK) << PLAB_SHIFT;
data |= ((100 * r->get_shared_allocs() / rs) & PERCENT_MASK) << SHARED_SHIFT;
+
+ data |= (r->age() & AGE_MASK) << AGE_SHIFT;
+ data |= (r->affiliation() & AFFILIATION_MASK) << AFFILIATION_SHIFT;
data |= (r->state_ordinal() & STATUS_MASK) << STATUS_SHIFT;
_regions_data[i]->set_value(data);
}
+
+ // If logging enabled, dump current region snapshot to log file
+ write_snapshot(_regions_data, _timestamp, _status, num_regions, rs >> 10, VERSION_NUMBER);
}
+ }
+ }
+}
+static int encode_phase(ShenandoahHeap* heap) {
+ if (heap->is_evacuation_in_progress() || heap->is_full_gc_move_in_progress()) {
+ return 2;
+ }
+ if (heap->is_update_refs_in_progress() || heap->is_full_gc_move_in_progress()) {
+ return 3;
+ }
+ if (heap->is_concurrent_mark_in_progress() || heap->is_full_gc_in_progress()) {
+ return 1;
+ }
+ assert(heap->is_idle(), "What is it doing?");
+ return 0;
+}
+
+static int get_generation_shift(ShenandoahGeneration* generation) {
+ switch (generation->type()) {
+ case NON_GEN:
+ case GLOBAL:
+ return 0;
+ case OLD:
+ return 2;
+ case YOUNG:
+ return 4;
+ default:
+ ShouldNotReachHere();
+ return -1;
+ }
+}
+
+jlong ShenandoahHeapRegionCounters::encode_heap_status(ShenandoahHeap* heap) {
+
+ if (heap->is_idle() && !heap->is_full_gc_in_progress()) {
+ return 0;
+ }
+
+ jlong status = 0;
+ if (!heap->mode()->is_generational()) {
+ status = encode_phase(heap);
+ } else {
+ int phase = encode_phase(heap);
+ ShenandoahGeneration* generation = heap->active_generation();
+ assert(generation != nullptr, "Expected active generation in this mode.");
+ int shift = get_generation_shift(generation);
+ status |= ((phase & 0x3) << shift);
+ if (heap->is_concurrent_old_mark_in_progress()) {
+ status |= (1 << 2);
}
+ log_develop_trace(gc)("%s, phase=%u, old_mark=%s, status=" JLONG_FORMAT,
+ generation->name(), phase, BOOL_TO_STR(heap->is_concurrent_old_mark_in_progress()), status);
}
+
+ if (heap->is_degenerated_gc_in_progress()) {
+ status |= (1 << 6);
+ }
+ if (heap->is_full_gc_in_progress()) {
+ status |= (1 << 7);
+ }
+
+ return status;
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp
index f0d4c2ad38f..c139980af41 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2016, 2019, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,6 +27,7 @@
#define SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGIONCOUNTERS_HPP
#include "memory/allocation.hpp"
+#include "logging/logFileStreamOutput.hpp"
/**
* This provides the following in JVMStat:
@@ -37,9 +39,14 @@
*
* variables:
* - sun.gc.shenandoah.regions.status current GC status:
- * - bit 0 set when marking in progress
- * - bit 1 set when evacuation in progress
- * - bit 2 set when update refs in progress
+ * | global | old | young | mode |
+ * | 0..1 | 2..3 | 4..5 | 6..7 |
+ *
+ * For each generation:
+ * 0 = idle, 1 = marking, 2 = evacuating, 3 = updating refs
+ *
+ * For mode:
+ * 0 = concurrent, 1 = degenerated, 2 = full
*
* two variable counters per region, with $max_regions (see above) counters:
* - sun.gc.shenandoah.regions.region.$i.data
@@ -51,24 +58,31 @@
* - bits 14-20 tlab allocated memory in percent
* - bits 21-27 gclab allocated memory in percent
* - bits 28-34 shared allocated memory in percent
- * - bits 35-41
+ * - bits 35-41 plab allocated memory in percent
* - bits 42-50
- * - bits 51-57
+ * - bits 51-55 age
+ * - bits 56-57 affiliation: 0 = free, young = 1, old = 2
* - bits 58-63 status
* - bits describe the state as recorded in ShenandoahHeapRegion
*/
class ShenandoahHeapRegionCounters : public CHeapObj {
private:
- static const jlong PERCENT_MASK = 0x7f;
- static const jlong STATUS_MASK = 0x3f;
+ static const jlong PERCENT_MASK = 0x7f;
+ static const jlong AGE_MASK = 0x1f;
+ static const jlong AFFILIATION_MASK = 0x03;
+ static const jlong STATUS_MASK = 0x3f;
- static const jlong USED_SHIFT = 0;
- static const jlong LIVE_SHIFT = 7;
- static const jlong TLAB_SHIFT = 14;
- static const jlong GCLAB_SHIFT = 21;
- static const jlong SHARED_SHIFT = 28;
+ static const jlong USED_SHIFT = 0;
+ static const jlong LIVE_SHIFT = 7;
+ static const jlong TLAB_SHIFT = 14;
+ static const jlong GCLAB_SHIFT = 21;
+ static const jlong SHARED_SHIFT = 28;
+ static const jlong PLAB_SHIFT = 35;
+ static const jlong AGE_SHIFT = 51;
+ static const jlong AFFILIATION_SHIFT = 56;
+ static const jlong STATUS_SHIFT = 58;
- static const jlong STATUS_SHIFT = 58;
+ static const jlong VERSION_NUMBER = 2;
char* _name_space;
PerfLongVariable** _regions_data;
@@ -76,10 +90,19 @@ class ShenandoahHeapRegionCounters : public CHeapObj {
PerfLongVariable* _status;
volatile jlong _last_sample_millis;
+ void write_snapshot(PerfLongVariable** regions,
+ PerfLongVariable* ts,
+ PerfLongVariable* status,
+ size_t num_regions,
+ size_t region_size, size_t protocolVersion);
+
public:
ShenandoahHeapRegionCounters();
~ShenandoahHeapRegionCounters();
void update();
+
+private:
+ static jlong encode_heap_status(ShenandoahHeap* heap) ;
};
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGIONCOUNTERS_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp
index 775b84a8966..686295c3e1b 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -27,16 +28,13 @@
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
+#include "gc/shenandoah/shenandoahGeneration.hpp"
#include "gc/shenandoah/shenandoahMark.inline.hpp"
#include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
#include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
#include "gc/shenandoah/shenandoahUtils.hpp"
#include "gc/shenandoah/shenandoahVerifier.hpp"
-ShenandoahMark::ShenandoahMark() :
- _task_queues(ShenandoahHeap::heap()->marking_context()->task_queues()) {
-}
-
void ShenandoahMark::start_mark() {
if (!CodeCache::is_gc_marking_cycle_active()) {
CodeCache::on_gc_marking_cycle_start();
@@ -46,34 +44,34 @@ void ShenandoahMark::start_mark() {
void ShenandoahMark::end_mark() {
// Unlike other GCs, we do not arm the nmethods
// when marking terminates.
- CodeCache::on_gc_marking_cycle_finish();
+ if (!ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress()) {
+ CodeCache::on_gc_marking_cycle_finish();
+ }
}
-void ShenandoahMark::clear() {
- // Clean up marking stacks.
- ShenandoahObjToScanQueueSet* queues = ShenandoahHeap::heap()->marking_context()->task_queues();
- queues->clear();
-
- // Cancel SATB buffers.
- ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking();
+ShenandoahMark::ShenandoahMark(ShenandoahGeneration* generation) :
+ _generation(generation),
+ _task_queues(generation->task_queues()),
+ _old_gen_task_queues(generation->old_gen_task_queues()) {
}
template
-void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahReferenceProcessor *rp, StringDedup::Requests* const req) {
+void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahReferenceProcessor *rp, StringDedup::Requests* const req, bool update_refs) {
ShenandoahObjToScanQueue* q = get_queue(w);
+ ShenandoahObjToScanQueue* old_q = get_old_queue(w);
ShenandoahHeap* const heap = ShenandoahHeap::heap();
ShenandoahLiveData* ld = heap->get_liveness_cache(w);
// TODO: We can clean up this if we figure out how to do templated oop closures that
// play nice with specialized_oop_iterators.
- if (heap->has_forwarded_objects()) {
+ if (update_refs) {
using Closure = ShenandoahMarkUpdateRefsClosure;
- Closure cl(q, rp);
+ Closure cl(q, rp, old_q);
mark_loop_work(&cl, ld, w, t, req);
} else {
using Closure = ShenandoahMarkRefsClosure;
- Closure cl(q, rp);
+ Closure cl(q, rp, old_q);
mark_loop_work(&cl, ld, w, t, req);
}
@@ -83,12 +81,29 @@ void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahRefe
template
void ShenandoahMark::mark_loop(uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp,
ShenandoahGenerationType generation, StringDedup::Requests* const req) {
- mark_loop_prework(worker_id, terminator, rp, req);
+ bool update_refs = ShenandoahHeap::heap()->has_forwarded_objects();
+ switch (generation) {
+ case YOUNG:
+ mark_loop_prework(worker_id, terminator, rp, req, update_refs);
+ break;
+ case OLD:
+ // Old generation collection only performs marking, it should not update references.
+ mark_loop_prework(worker_id, terminator, rp, req, false);
+ break;
+ case GLOBAL:
+ mark_loop_prework(worker_id, terminator, rp, req, update_refs);
+ break;
+ case NON_GEN:
+ mark_loop_prework(worker_id, terminator, rp, req, update_refs);
+ break;
+ default:
+ ShouldNotReachHere();
+ break;
+ }
}
void ShenandoahMark::mark_loop(uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp,
- ShenandoahGenerationType generation, bool cancellable, StringDedupMode dedup_mode,
- StringDedup::Requests* const req) {
+ ShenandoahGenerationType generation, bool cancellable, StringDedupMode dedup_mode, StringDedup::Requests* const req) {
if (cancellable) {
switch(dedup_mode) {
case NO_DEDUP:
@@ -125,7 +140,12 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w
ShenandoahObjToScanQueue* q;
ShenandoahMarkTask t;
- heap->ref_processor()->set_mark_closure(worker_id, cl);
+ // Do not use active_generation() : we must use the gc_generation() set by
+ // ShenandoahGCScope on the ControllerThread's stack; no safepoint may
+ // intervene to update active_generation, so we can't
+ // shenandoah_assert_generations_reconciled() here.
+ assert(heap->gc_generation()->type() == GENERATION, "Sanity: %d != %d", heap->gc_generation()->type(), GENERATION);
+ heap->gc_generation()->ref_processor()->set_mark_closure(worker_id, cl);
/*
* Process outstanding queues, if any.
@@ -145,7 +165,7 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w
for (uint i = 0; i < stride; i++) {
if (q->pop(t)) {
- do_task(q, cl, live_data, req, &t);
+ do_task(q, cl, live_data, req, &t, worker_id);
} else {
assert(q->is_empty(), "Must be empty");
q = queues->claim_next();
@@ -154,8 +174,9 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w
}
}
q = get_queue(worker_id);
+ ShenandoahObjToScanQueue* old_q = get_old_queue(worker_id);
- ShenandoahSATBBufferClosure drain_satb(q);
+ ShenandoahSATBBufferClosure drain_satb(q, old_q);
SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
/*
@@ -165,7 +186,6 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w
if (CANCELLABLE && heap->check_cancelled_gc_and_yield()) {
return;
}
-
while (satb_mq_set.completed_buffers_num() > 0) {
satb_mq_set.apply_closure_to_completed_buffer(&drain_satb);
}
@@ -174,7 +194,7 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w
for (uint i = 0; i < stride; i++) {
if (q->pop(t) ||
queues->steal(worker_id, t)) {
- do_task(q, cl, live_data, req, &t);
+ do_task(q, cl, live_data, req, &t, worker_id);
work++;
} else {
break;
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp
index fab913bfd94..ae8d52a3d0e 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,10 +26,12 @@
#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHMARK_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHMARK_HPP
+#include "gc/shared/ageTable.hpp"
#include "gc/shared/stringdedup/stringDedup.hpp"
#include "gc/shared/taskTerminator.hpp"
#include "gc/shenandoah/shenandoahGenerationType.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahGeneration.hpp"
#include "gc/shenandoah/shenandoahTaskqueue.hpp"
enum StringDedupMode {
@@ -45,16 +48,16 @@ class ShenandoahReferenceProcessor;
// maintained by task queues, mark bitmap and SATB buffers (concurrent mark)
class ShenandoahMark: public StackObj {
protected:
+ ShenandoahGeneration* const _generation;
ShenandoahObjToScanQueueSet* const _task_queues;
+ ShenandoahObjToScanQueueSet* const _old_gen_task_queues;
protected:
- ShenandoahMark();
+ ShenandoahMark(ShenandoahGeneration* generation);
public:
template
- static inline void mark_through_ref(T* p, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context, bool weak);
-
- static void clear();
+ static inline void mark_through_ref(T* p, ShenandoahObjToScanQueue* q, ShenandoahObjToScanQueue* old_q, ShenandoahMarkingContext* const mark_context, bool weak);
// Loom support
void start_mark();
@@ -62,12 +65,20 @@ class ShenandoahMark: public StackObj {
// Helpers
inline ShenandoahObjToScanQueueSet* task_queues() const;
+ ShenandoahObjToScanQueueSet* old_task_queues() {
+ return _old_gen_task_queues;
+ }
+
inline ShenandoahObjToScanQueue* get_queue(uint index) const;
+ inline ShenandoahObjToScanQueue* get_old_queue(uint index) const;
+
+ inline ShenandoahGeneration* generation() { return _generation; };
-// ---------- Marking loop and tasks
private:
+// ---------- Marking loop and tasks
+
template
- inline void do_task(ShenandoahObjToScanQueue* q, T* cl, ShenandoahLiveData* live_data, StringDedup::Requests* const req, ShenandoahMarkTask* task);
+ inline void do_task(ShenandoahObjToScanQueue* q, T* cl, ShenandoahLiveData* live_data, StringDedup::Requests* const req, ShenandoahMarkTask* task, uint worker_id);
template
inline void do_chunked_array_start(ShenandoahObjToScanQueue* q, T* cl, oop array, bool weak);
@@ -76,13 +87,23 @@ class ShenandoahMark: public StackObj {
inline void do_chunked_array(ShenandoahObjToScanQueue* q, T* cl, oop array, int chunk, int pow, bool weak);
template
- inline void count_liveness(ShenandoahLiveData* live_data, oop obj);
+ inline void count_liveness(ShenandoahLiveData* live_data, oop obj, uint worker_id);
template
void mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint worker_id, TaskTerminator *t, StringDedup::Requests* const req);
template
- void mark_loop_prework(uint worker_id, TaskTerminator *terminator, ShenandoahReferenceProcessor *rp, StringDedup::Requests* const req);
+ void mark_loop_prework(uint worker_id, TaskTerminator *terminator, ShenandoahReferenceProcessor *rp, StringDedup::Requests* const req, bool update_refs);
+
+ template
+ static bool in_generation(ShenandoahHeap* const heap, oop obj);
+
+ template
+ static void mark_non_generational_ref(T *p, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context, bool weak);
+
+ static void mark_ref(ShenandoahObjToScanQueue* q,
+ ShenandoahMarkingContext* const mark_context,
+ bool weak, oop obj);
template
inline void dedup_string(oop obj, StringDedup::Requests* const req);
@@ -90,6 +111,7 @@ class ShenandoahMark: public StackObj {
template
void mark_loop(uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp,
ShenandoahGenerationType generation, StringDedup::Requests* const req);
+
void mark_loop(uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp,
ShenandoahGenerationType generation, bool cancellable, StringDedupMode dedup_mode, StringDedup::Requests* const req);
};
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp
index 2eca17bde27..0239f961c65 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2015, 2022, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,11 +29,14 @@
#include "gc/shenandoah/shenandoahMark.hpp"
#include "gc/shared/continuationGCSupport.inline.hpp"
+#include "gc/shenandoah/shenandoahAgeCensus.hpp"
#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
+#include "gc/shenandoah/shenandoahOldGeneration.hpp"
+#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
#include "gc/shenandoah/shenandoahStringDedup.inline.hpp"
#include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
#include "gc/shenandoah/shenandoahUtils.hpp"
@@ -58,7 +62,7 @@ void ShenandoahMark::dedup_string(oop obj, StringDedup::Requests* const req) {
}
template
-void ShenandoahMark::do_task(ShenandoahObjToScanQueue* q, T* cl, ShenandoahLiveData* live_data, StringDedup::Requests* const req, ShenandoahMarkTask* task) {
+void ShenandoahMark::do_task(ShenandoahObjToScanQueue* q, T* cl, ShenandoahLiveData* live_data, StringDedup::Requests* const req, ShenandoahMarkTask* task, uint worker_id) {
oop obj = task->obj();
shenandoah_assert_not_forwarded(nullptr, obj);
@@ -95,7 +99,7 @@ void ShenandoahMark::do_task(ShenandoahObjToScanQueue* q, T* cl, ShenandoahLiveD
// Avoid double-counting objects that are visited twice due to upgrade
// from final- to strong mark.
if (task->count_liveness()) {
- count_liveness(live_data, obj);
+ count_liveness(live_data, obj, worker_id);
}
} else {
// Case 4: Array chunk, has sensible chunk id. Process it.
@@ -104,14 +108,27 @@ void ShenandoahMark::do_task(ShenandoahObjToScanQueue* q, T* cl, ShenandoahLiveD
}
template
-inline void ShenandoahMark::count_liveness(ShenandoahLiveData* live_data, oop obj) {
- ShenandoahHeap* const heap = ShenandoahHeap::heap();
- size_t region_idx = heap->heap_region_index_containing(obj);
- ShenandoahHeapRegion* region = heap->get_region(region_idx);
- size_t size = obj->size();
+inline void ShenandoahMark::count_liveness(ShenandoahLiveData* live_data, oop obj, uint worker_id) {
+ const ShenandoahHeap* const heap = ShenandoahHeap::heap();
+ const size_t region_idx = heap->heap_region_index_containing(obj);
+ ShenandoahHeapRegion* const region = heap->get_region(region_idx);
+ const size_t size = obj->size();
+
+ // Age census for objects in the young generation
+ if (GENERATION == YOUNG || (GENERATION == GLOBAL && region->is_young())) {
+ assert(heap->mode()->is_generational(), "Only if generational");
+ if (ShenandoahGenerationalAdaptiveTenuring && !ShenandoahGenerationalCensusAtEvac) {
+ assert(region->is_young(), "Only for young objects");
+ uint age = ShenandoahHeap::get_object_age(obj);
+ ShenandoahAgeCensus* const census = ShenandoahGenerationalHeap::heap()->age_census();
+ CENSUS_NOISE(census->add(age, region->age(), region->youth(), size, worker_id);)
+ NO_CENSUS_NOISE(census->add(age, region->age(), size, worker_id);)
+ }
+ }
if (!region->is_humongous_start()) {
assert(!region->is_humongous(), "Cannot have continuations here");
+ assert(region->is_affiliated(), "Do not count live data within Free Regular Region " SIZE_FORMAT, region_idx);
ShenandoahLiveData cur = live_data[region_idx];
size_t new_val = size + cur;
if (new_val >= SHENANDOAH_LIVEDATA_MAX) {
@@ -126,9 +143,11 @@ inline void ShenandoahMark::count_liveness(ShenandoahLiveData* live_data, oop ob
shenandoah_assert_in_correct_region(nullptr, obj);
size_t num_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
+ assert(region->is_affiliated(), "Do not count live data within FREE Humongous Start Region " SIZE_FORMAT, region_idx);
for (size_t i = region_idx; i < region_idx + num_regions; i++) {
ShenandoahHeapRegion* chain_reg = heap->get_region(i);
assert(chain_reg->is_humongous(), "Expecting a humongous region");
+ assert(chain_reg->is_affiliated(), "Do not count live data within FREE Humongous Continuation Region " SIZE_FORMAT, i);
chain_reg->increase_live_data_gc_words(chain_reg->used() >> LogHeapWordSize);
}
}
@@ -235,50 +254,121 @@ template
class ShenandoahSATBBufferClosure : public SATBBufferClosure {
private:
ShenandoahObjToScanQueue* _queue;
+ ShenandoahObjToScanQueue* _old_queue;
ShenandoahHeap* _heap;
ShenandoahMarkingContext* const _mark_context;
public:
- ShenandoahSATBBufferClosure(ShenandoahObjToScanQueue* q) :
+ ShenandoahSATBBufferClosure(ShenandoahObjToScanQueue* q, ShenandoahObjToScanQueue* old_q) :
_queue(q),
+ _old_queue(old_q),
_heap(ShenandoahHeap::heap()),
_mark_context(_heap->marking_context())
{
}
void do_buffer(void **buffer, size_t size) {
- assert(size == 0 || !_heap->has_forwarded_objects(), "Forwarded objects are not expected here");
+ assert(size == 0 || !_heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded objects are not expected here");
for (size_t i = 0; i < size; ++i) {
oop *p = (oop *) &buffer[i];
- ShenandoahMark::mark_through_ref