diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index ceedb4f10632a..ac2acfba92353 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -3306,7 +3306,7 @@ void MacroAssembler::subw(Register Rd, Register Rn, RegisterOrConstant decrement void MacroAssembler::reinit_heapbase() { if (UseCompressedOops) { - if (Universe::is_fully_initialized()) { + if (Universe::is_fully_initialized() && !AOTCodeCache::is_on_for_dump()) { mov(rheapbase, CompressedOops::base()); } else { lea(rheapbase, ExternalAddress(CompressedOops::base_addr())); diff --git a/src/hotspot/cpu/aarch64/runtime_aarch64.cpp b/src/hotspot/cpu/aarch64/runtime_aarch64.cpp index d45f9865bd26a..2e570bee7091c 100644 --- a/src/hotspot/cpu/aarch64/runtime_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/runtime_aarch64.cpp @@ -292,7 +292,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() { assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned"); const char* name = OptoRuntime::stub_name(StubId::c2_exception_id); - CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::C2Blob, (uint)BlobId::c2_exception_id, name); + CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::C2Blob, BlobId::c2_exception_id); if (blob != nullptr) { return blob->as_exception_blob(); } diff --git a/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp b/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp index 695534604b8c0..95b59bd8590f5 100644 --- a/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp @@ -84,8 +84,7 @@ do_stub(compiler, count_positives) \ do_arch_entry(aarch64, compiler, count_positives, count_positives, \ count_positives) \ - do_stub(compiler, count_positives_long) \ - do_arch_entry(aarch64, compiler, count_positives_long, \ + do_arch_entry(aarch64, compiler, count_positives, \ count_positives_long, count_positives_long) \ do_stub(compiler, compare_long_string_LL) \ do_arch_entry(aarch64, compiler, compare_long_string_LL, \ @@ -108,8 +107,9 @@ do_stub(compiler, string_indexof_linear_ul) \ do_arch_entry(aarch64, compiler, string_indexof_linear_ul, \ string_indexof_linear_ul, string_indexof_linear_ul) \ - /* this uses the entry for ghash_processBlocks */ \ - do_stub(compiler, ghash_processBlocks_wide) \ + do_stub(compiler, ghash_processBlocks_small) \ + do_arch_entry(aarch64, compiler, ghash_processBlocks_small, \ + ghash_processBlocks_small, ghash_processBlocks_small) \ #define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \ @@ -142,6 +142,47 @@ /* stub only -- entries are not stored in StubRoutines::aarch64 */ \ /* n.b. these are not the same as the generic atomic stubs */ \ do_stub(final, atomic_entry_points) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_fetch_add_4_impl, atomic_fetch_add_4_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_fetch_add_8_impl, atomic_fetch_add_8_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_fetch_add_4_relaxed_impl, \ + atomic_fetch_add_4_relaxed_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_fetch_add_8_relaxed_impl, \ + atomic_fetch_add_8_relaxed_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_xchg_4_impl, atomic_xchg_4_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_xchg_8_impl, atomic_xchg_8_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_1_impl, atomic_cmpxchg_1_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_4_impl, atomic_cmpxchg_4_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_8_impl, atomic_cmpxchg_8_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_1_relaxed_impl, \ + atomic_cmpxchg_1_relaxed_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_4_relaxed_impl, \ + atomic_cmpxchg_4_relaxed_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_8_relaxed_impl, \ + atomic_cmpxchg_8_relaxed_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_4_release_impl, \ + atomic_cmpxchg_4_release_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_8_release_impl, \ + atomic_cmpxchg_8_release_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_4_seq_cst_impl, \ + atomic_cmpxchg_4_seq_cst_impl) \ + do_arch_entry(aarch64, final, atomic_entry_points, \ + atomic_cmpxchg_8_seq_cst_impl, \ + atomic_cmpxchg_8_seq_cst_impl) \ #endif // CPU_AARCH64_STUBDECLARATIONS_HPP diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 8dbc5dbac0343..266bd7097144b 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -79,6 +79,166 @@ #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") +// Constant data definitions + +static const uint32_t _sha256_round_consts[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +}; + +static const uint64_t _sha512_round_consts[80] = { + 0x428A2F98D728AE22L, 0x7137449123EF65CDL, 0xB5C0FBCFEC4D3B2FL, + 0xE9B5DBA58189DBBCL, 0x3956C25BF348B538L, 0x59F111F1B605D019L, + 0x923F82A4AF194F9BL, 0xAB1C5ED5DA6D8118L, 0xD807AA98A3030242L, + 0x12835B0145706FBEL, 0x243185BE4EE4B28CL, 0x550C7DC3D5FFB4E2L, + 0x72BE5D74F27B896FL, 0x80DEB1FE3B1696B1L, 0x9BDC06A725C71235L, + 0xC19BF174CF692694L, 0xE49B69C19EF14AD2L, 0xEFBE4786384F25E3L, + 0x0FC19DC68B8CD5B5L, 0x240CA1CC77AC9C65L, 0x2DE92C6F592B0275L, + 0x4A7484AA6EA6E483L, 0x5CB0A9DCBD41FBD4L, 0x76F988DA831153B5L, + 0x983E5152EE66DFABL, 0xA831C66D2DB43210L, 0xB00327C898FB213FL, + 0xBF597FC7BEEF0EE4L, 0xC6E00BF33DA88FC2L, 0xD5A79147930AA725L, + 0x06CA6351E003826FL, 0x142929670A0E6E70L, 0x27B70A8546D22FFCL, + 0x2E1B21385C26C926L, 0x4D2C6DFC5AC42AEDL, 0x53380D139D95B3DFL, + 0x650A73548BAF63DEL, 0x766A0ABB3C77B2A8L, 0x81C2C92E47EDAEE6L, + 0x92722C851482353BL, 0xA2BFE8A14CF10364L, 0xA81A664BBC423001L, + 0xC24B8B70D0F89791L, 0xC76C51A30654BE30L, 0xD192E819D6EF5218L, + 0xD69906245565A910L, 0xF40E35855771202AL, 0x106AA07032BBD1B8L, + 0x19A4C116B8D2D0C8L, 0x1E376C085141AB53L, 0x2748774CDF8EEB99L, + 0x34B0BCB5E19B48A8L, 0x391C0CB3C5C95A63L, 0x4ED8AA4AE3418ACBL, + 0x5B9CCA4F7763E373L, 0x682E6FF3D6B2B8A3L, 0x748F82EE5DEFB2FCL, + 0x78A5636F43172F60L, 0x84C87814A1F0AB72L, 0x8CC702081A6439ECL, + 0x90BEFFFA23631E28L, 0xA4506CEBDE82BDE9L, 0xBEF9A3F7B2C67915L, + 0xC67178F2E372532BL, 0xCA273ECEEA26619CL, 0xD186B8C721C0C207L, + 0xEADA7DD6CDE0EB1EL, 0xF57D4F7FEE6ED178L, 0x06F067AA72176FBAL, + 0x0A637DC5A2C898A6L, 0x113F9804BEF90DAEL, 0x1B710B35131C471BL, + 0x28DB77F523047D84L, 0x32CAAB7B40C72493L, 0x3C9EBE0A15C9BEBCL, + 0x431D67C49C100D4CL, 0x4CC5D4BECB3E42B6L, 0x597F299CFC657E2AL, + 0x5FCB6FAB3AD6FAECL, 0x6C44198C4A475817L +}; + +static const uint64_t _sha3_round_consts[24] = { + 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL, + 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L, + 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL, + 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL, + 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L, + 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L, + 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L, + 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L +}; + +static const uint64_t _double_keccak_round_consts[24] = { + 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL, + 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L, + 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL, + 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL, + 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L, + 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L, + 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L, + 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L +}; + +static const char _encodeBlock_toBase64[64] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/' +}; + +static const char _encodeBlock_toBase64URL[64] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_' +}; + +// Non-SIMD lookup tables are mostly dumped from fromBase64 array used in java.util.Base64, +// except the trailing character '=' is also treated illegal value in this intrinsic. That +// is java.util.Base64.fromBase64['='] = -2, while fromBase(URL)64ForNoSIMD['='] = 255 here. +static const uint8_t _decodeBlock_fromBase64ForNoSIMD[256] = { + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u, + 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, + 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 255u, + 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u, + 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, +}; + +static const uint8_t _decodeBlock_fromBase64URLForNoSIMD[256] = { + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, + 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, + 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 63u, + 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u, + 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, +}; + +// A legal value of base64 code is in range [0, 127]. We need two lookups +// with tbl/tbx and combine them to get the decode data. The 1st table vector +// lookup use tbl, out of range indices are set to 0 in destination. The 2nd +// table vector lookup use tbx, out of range indices are unchanged in +// destination. Input [64..126] is mapped to index [65, 127] in second lookup. +// The value of index 64 is set to 0, so that we know that we already get the +// decoded data with the 1st lookup. +static const uint8_t _decodeBlock_fromBase64ForSIMD[128] = { + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u, + 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, + 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, + 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, + 255u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, + 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, +}; + +static const uint8_t _decodeBlock_fromBase64URLForSIMD[128] = { + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, + 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, + 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, + 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, + 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, + 63u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, + 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, +}; + + // Stub Code definitions class StubGenerator: public StubCodeGenerator { @@ -203,6 +363,19 @@ class StubGenerator: public StubCodeGenerator { "adjust this code"); StubId stub_id = StubId::stubgen_call_stub_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 2, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + // this stub has an extra return entry which we need to retrieve + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == 1, "expected 1 extra entry"); + return_address = entries.at(0); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -323,6 +496,7 @@ class StubGenerator: public StubCodeGenerator { // save current address for use by exception handling code return_address = __ pc(); + entries.append(return_address); // store result depending on type (everything that is not // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT) @@ -406,6 +580,9 @@ class StubGenerator: public StubCodeGenerator { __ strd(j_farg0, Address(j_rarg2, 0)); __ br(Assembler::AL, exit); + // record the stub entry and end plus the auxiliary entry + store_archive_data(stub_id, start, __ pc(), &entries); + return start; } @@ -423,6 +600,15 @@ class StubGenerator: public StubCodeGenerator { address generate_catch_exception() { StubId stub_id = StubId::stubgen_catch_exception_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -450,7 +636,9 @@ class StubGenerator: public StubCodeGenerator { __ verify_oop(r0); __ str(r0, Address(rthread, Thread::pending_exception_offset())); - __ mov(rscratch1, (address)__FILE__); + // special case -- add file name string to AOT address table + address file = (address)AOTCodeCache::add_C_string(__FILE__); + __ lea(rscratch1, ExternalAddress(file)); __ str(rscratch1, Address(rthread, Thread::exception_file_offset())); __ movw(rscratch1, (int)__LINE__); __ strw(rscratch1, Address(rthread, Thread::exception_line_offset())); @@ -460,6 +648,9 @@ class StubGenerator: public StubCodeGenerator { "_call_stub_return_address must have been generated before"); __ b(StubRoutines::_call_stub_return_address); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -479,6 +670,14 @@ class StubGenerator: public StubCodeGenerator { address generate_forward_exception() { StubId stub_id = StubId::stubgen_forward_exception_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -551,6 +750,9 @@ class StubGenerator: public StubCodeGenerator { __ verify_oop(r0); __ br(r19); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -569,6 +771,15 @@ class StubGenerator: public StubCodeGenerator { // [tos + 5]: saved rscratch1 address generate_verify_oop() { StubId stub_id = StubId::stubgen_verify_oop_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -613,11 +824,23 @@ class StubGenerator: public StubCodeGenerator { __ blr(rscratch1); __ hlt(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } // Generate indices for iota vector. address generate_iota_indices(StubId stub_id) { + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -639,6 +862,10 @@ class StubGenerator: public StubCodeGenerator { // D - FP __ emit_data64(0x0000000000000000, relocInfo::none); // 0.0d __ emit_data64(0x3FF0000000000000, relocInfo::none); // 1.0d + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -656,14 +883,23 @@ class StubGenerator: public StubCodeGenerator { // r11 < MacroAssembler::zero_words_block_size. address generate_zero_blocks() { + StubId stub_id = StubId::stubgen_zero_blocks_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + StubCodeMark mark(this, stub_id); Label done; Label base_aligned; Register base = r10, cnt = r11; __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_zero_blocks_id; - StubCodeMark mark(this, stub_id); address start = __ pc(); if (UseBlockZeroing) { @@ -707,6 +943,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -803,6 +1042,15 @@ class StubGenerator: public StubCodeGenerator { // s and d are adjusted to point to the remaining words to copy // address generate_copy_longs(StubId stub_id, DecoratorSet decorators, Register s, Register d, Register count) { + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + BasicType type; copy_direction direction; @@ -1154,6 +1402,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); } + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1508,8 +1759,8 @@ class StubGenerator: public StubCodeGenerator { // stub_id - is used to name the stub and identify all details of // how to perform the copy. // - // entry - is assigned to the stub's post push entry point unless - // it is null + // nopush_entry - is assigned to the stub's post push entry point + // unless it is null // // Inputs: // c_rarg0 - source array address @@ -1525,8 +1776,6 @@ class StubGenerator: public StubCodeGenerator { // copy method // address generate_disjoint_copy(StubId stub_id, address *nopush_entry) { - Register s = c_rarg0, d = c_rarg1, count = c_rarg2; - RegSet saved_reg = RegSet::of(s, d, count); int size; bool aligned; bool is_oop; @@ -1607,17 +1856,48 @@ class StubGenerator: public StubCodeGenerator { ShouldNotReachHere(); break; } + // all stubs provide a 2nd entry which omits the frame push for + // use when bailing out from a conjoint copy. However we may also + // need some extra addressses for memory access protection. + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 2, "sanity check"); + assert(nopush_entry != nullptr, "all disjoint copy stubs export a nopush entry"); + + bool add_extras = !is_oop && (!aligned || sizeof(jlong) == size); + int extra_count = ((add_extras ? 1 : 0) * 3); + GrowableArray
entries; + GrowableArray
extras; + GrowableArray
*extras_ptr = (extra_count > 0 ? &extras : nullptr); + + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries, extras_ptr); + assert(entries.length() == entry_count - 1, + "unexpected entries count %d", entries.length()); + *nopush_entry = entries.at(0); + assert(extras.length() == extra_count, + "unexpected extra count %d", extras.length()); + if (add_extras) { + // register one handler at offset 0 + register_unsafe_access_handlers(extras, 0, 1, end); + } + return start; + } + + Register s = c_rarg0, d = c_rarg1, count = c_rarg2; + RegSet saved_reg = RegSet::of(s, d, count); __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); - if (nopush_entry != nullptr) { - *nopush_entry = __ pc(); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - BLOCK_COMMENT("Entry:"); - } + *nopush_entry = __ pc(); + entries.append(*nopush_entry); + + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Post-Push Entry:"); DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; if (dest_uninitialized) { @@ -1636,8 +1916,7 @@ class StubGenerator: public StubCodeGenerator { } { // UnsafeMemoryAccess page error: continue after unsafe access - bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size); - UnsafeMemoryAccessMark umam(this, add_entry, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, size); } @@ -1652,6 +1931,20 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ mov(r0, zr); // return 0 __ ret(lr); + + address end = __ pc(); + + if (add_extras) { + // retrieve the registered handler addresses + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == extra_count + , "incorrect handlers count %d", extras.length()); + } + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, &entries, extras_ptr); + return start; } @@ -1663,8 +1956,8 @@ class StubGenerator: public StubCodeGenerator { // corresponding disjoint copy routine which can be // jumped to if the ranges do not actually overlap // - // entry - is assigned to the stub's post push entry point unless - // it is null + // nopush_entry - is assigned to the stub's post push entry point + // unless it is null // // // Inputs: @@ -1681,8 +1974,6 @@ class StubGenerator: public StubCodeGenerator { // used by some other conjoint copy method // address generate_conjoint_copy(StubId stub_id, address nooverlap_target, address *nopush_entry) { - Register s = c_rarg0, d = c_rarg1, count = c_rarg2; - RegSet saved_regs = RegSet::of(s, d, count); int size; bool aligned; bool is_oop; @@ -1762,15 +2053,50 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } + // only some conjoint stubs generate a 2nd entry + int entry_count = StubInfo::entry_count(stub_id); + int expected_entry_count = (nopush_entry == nullptr ? 1 : 2); + assert(entry_count == expected_entry_count, + "expected entry count %d does not match declared entry count %d for stub %s", + expected_entry_count, entry_count, StubInfo::name(stub_id)); + + // We need to protect memory accesses in certain cases + bool add_extras = !is_oop && (!aligned || sizeof(jlong) == size); + int extra_count = ((add_extras ? 1 : 0) * 3); + GrowableArray
entries; + GrowableArray
extras; + GrowableArray
*entries_ptr = (nopush_entry != nullptr ? &entries : nullptr); + GrowableArray
*extras_ptr = (extra_count > 0 ? &extras : nullptr); + + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + assert(entries.length() == expected_entry_count - 1, + "unexpected entries count %d", entries.length()); + assert(extras.length() == extra_count, + "unexpected extra count %d", extras.length()); + if (nopush_entry != nullptr) { + *nopush_entry = entries.at(0); + } + if (add_extras) { + // register one handler at offset 0 + register_unsafe_access_handlers(extras, 0, 1, end); + } + return start; + } + Register s = c_rarg0, d = c_rarg1, count = c_rarg2; + RegSet saved_regs = RegSet::of(s, d, count); StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); if (nopush_entry != nullptr) { *nopush_entry = __ pc(); + entries.append(*nopush_entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - BLOCK_COMMENT("Entry:"); + BLOCK_COMMENT("Post-Push Entry:"); } // use fwd copy when (d-s) above_equal (count*size) @@ -1798,8 +2124,7 @@ class StubGenerator: public StubCodeGenerator { } { // UnsafeMemoryAccess page error: continue after unsafe access - bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size); - UnsafeMemoryAccessMark umam(this, add_entry, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, -size); } if (is_oop) { @@ -1811,6 +2136,23 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ mov(r0, zr); // return 0 __ ret(lr); + + assert(entries.length() == expected_entry_count - 1, + "unexpected entries count %d", entries.length()); + + address end = __ pc(); + + if (add_extras) { + // retrieve the registered handler addresses + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == extra_count, + "incorrect handlers count %d", extras.length()); + } + + // record the stub entry and end plus any no_push entry and/or + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + return start; } @@ -1864,6 +2206,30 @@ class StubGenerator: public StubCodeGenerator { ShouldNotReachHere(); } + // the normal stub provides a 2nd entry which omits the frame push + // for use when bailing out from a disjoint copy + // We need to protect memory accesses in certain cases + // only some conjoint stubs generate a 2nd entry + int entry_count = StubInfo::entry_count(stub_id); + int expected_entry_count = (nopush_entry == nullptr ? 1 : 2); + GrowableArray
entries; + GrowableArray
*entries_ptr = (expected_entry_count == 1 ? nullptr : &entries); + assert(entry_count == expected_entry_count, + "expected entry count %d does not match declared entry count %d for stub %s", + expected_entry_count, entry_count, StubInfo::name(stub_id)); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr); + assert(entries.length() + 1 == expected_entry_count, + "expected entry count %d does not match return entry count %d for stub %s", + expected_entry_count, entries.length() + 1, StubInfo::name(stub_id)); + if (nopush_entry != nullptr) { + *nopush_entry = entries.at(0); + } + return start; + } + Label L_load_element, L_store_element, L_do_card_marks, L_done, L_done_pop; // Input registers (after setup_arg_regs) @@ -1913,6 +2279,7 @@ class StubGenerator: public StubCodeGenerator { // Caller of this entry point must set up the argument registers. if (nopush_entry != nullptr) { *nopush_entry = __ pc(); + entries.append(*nopush_entry); BLOCK_COMMENT("Entry:"); } @@ -2010,6 +2377,8 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end plus any no_push entry + store_archive_data(stub_id, start, __ pc() , entries_ptr); return start; } @@ -2072,6 +2441,17 @@ class StubGenerator: public StubCodeGenerator { address int_copy_entry, address long_copy_entry) { StubId stub_id = StubId::stubgen_unsafe_arraycopy_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + // TODO - in debug mode we should save and restore the supplied + // copy entries and assert that the restored entries match the ones + // passed in + load_archive_data(stub_id, start, end); + return start; + } Label L_long_aligned, L_int_aligned, L_short_aligned; Register s = c_rarg0, d = c_rarg1, count = c_rarg2; @@ -2104,6 +2484,9 @@ class StubGenerator: public StubCodeGenerator { __ lsr(count, count, LogBytesPerLong); // size => long_count __ b(RuntimeAddress(long_copy_entry)); + // record the stub entry and end plus any no_push entry + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2125,6 +2508,17 @@ class StubGenerator: public StubCodeGenerator { address int_copy_entry, address oop_copy_entry, address long_copy_entry, address checkcast_copy_entry) { StubId stub_id = StubId::stubgen_generic_arraycopy_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + // TODO - in debug mode we should save and restore the supplied + // copy entries and assert that the restored entries match the ones + // passed in + load_archive_data(stub_id, start, end); + return start; + } Label L_failed, L_objArray; Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs; @@ -2383,6 +2777,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(lr); + // record the stub entry and end plus any no_push entry + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2427,6 +2824,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -2563,14 +2968,31 @@ class StubGenerator: public StubCodeGenerator { __ bind(L_exit2); __ leave(); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address generate_unsafecopy_common_error_exit() { + StubId stub_id = StubId::stubgen_unsafecopy_common_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } address start_pc = __ pc(); __ leave(); __ mov(r0, 0); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, start_pc, __ pc()); + return start_pc; } @@ -2589,13 +3011,30 @@ class StubGenerator: public StubCodeGenerator { // c_rarg2 - byte value // address generate_unsafe_setmemory() { + StubId stub_id = StubId::stubgen_unsafe_setmemory_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + // we expect one set of extra unsafememory access handler entries + GrowableArray
extras; + int extra_count = 3; + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, nullptr, &extras); + assert(extras.length() == extra_count, + "unexpected extra entry count %d", extras.length()); + register_unsafe_access_handlers(extras, 0, 1, end); + return start; + } + __ align(CodeEntryAlignment); - StubCodeMark mark(this, StubId::stubgen_unsafe_setmemory_id); + StubCodeMark mark(this, stub_id); address start = __ pc(); Register dest = c_rarg0, count = c_rarg1, value = c_rarg2; Label tail; + { UnsafeMemoryAccessMark umam(this, true, false); __ enter(); // required for proper stackwalking of RuntimeStub frame @@ -2679,6 +3118,17 @@ class StubGenerator: public StubCodeGenerator { __ bind(finished); __ leave(); __ ret(lr); + // have to exit the block and destroy the UnsafeMemoryAccessMark + // in order to retrieve the handler end address + } + + // install saved handler addresses in extras + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == extra_count, + "incorrect handlers count %d", extras.length()); + // record the stub entry and end plus the extras + store_archive_data(stub_id, start, end, nullptr, &extras); return start; } @@ -2686,9 +3136,18 @@ class StubGenerator: public StubCodeGenerator { address generate_data_cache_writeback() { const Register line = c_rarg0; // address of line to write back + StubId stub_id = StubId::stubgen_data_cache_writeback_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_data_cache_writeback_id; StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2697,15 +3156,27 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address generate_data_cache_writeback_sync() { + StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + const Register is_pre = c_rarg0; // pre or post sync __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id; StubCodeMark mark(this, stub_id); // pre wbsync is a no-op @@ -2720,6 +3191,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2882,8 +3356,18 @@ class StubGenerator: public StubCodeGenerator { // c_rarg2 - K (key) in little endian int array // address generate_aescrypt_encryptBlock() { - __ align(CodeEntryAlignment); + assert(UseAES, "need AES cryptographic extension support"); StubId stub_id = StubId::stubgen_aescrypt_encryptBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); const Register from = c_rarg0; // source array address @@ -2904,6 +3388,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2916,8 +3403,17 @@ class StubGenerator: public StubCodeGenerator { // address generate_aescrypt_decryptBlock() { assert(UseAES, "need AES cryptographic extension support"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_aescrypt_decryptBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); Label L_doLast; @@ -2938,6 +3434,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2955,8 +3454,17 @@ class StubGenerator: public StubCodeGenerator { // address generate_cipherBlockChaining_encryptAESCrypt() { assert(UseAES, "need AES cryptographic extension support"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_cipherBlockChaining_encryptAESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52; @@ -3043,6 +3551,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3060,8 +3571,17 @@ class StubGenerator: public StubCodeGenerator { // address generate_cipherBlockChaining_decryptAESCrypt() { assert(UseAES, "need AES cryptographic extension support"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52; @@ -3152,6 +3672,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3188,6 +3711,16 @@ class StubGenerator: public StubCodeGenerator { // r0 - input length // address generate_counterMode_AESCrypt() { + StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + const Register in = c_rarg0; const Register out = c_rarg1; const Register key = c_rarg2; @@ -3248,7 +3781,6 @@ class StubGenerator: public StubCodeGenerator { // Wide bulk encryption of whole blocks. __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id; StubCodeMark mark(this, stub_id); const address start = __ pc(); __ enter(); @@ -3435,6 +3967,9 @@ class StubGenerator: public StubCodeGenerator { __ strw(used, Address(used_ptr)); __ b(large_block_return); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3451,9 +3986,17 @@ class StubGenerator: public StubCodeGenerator { // return - number of processed bytes address generate_galoisCounterMode_AESCrypt() { Label ghash_polynomial; // local data generated after code - - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -3567,6 +4110,9 @@ class StubGenerator: public StubCodeGenerator { // 128-bit vector __ emit_int64(0x87); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3685,6 +4231,15 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -3815,6 +4370,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3838,6 +4396,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); @@ -3919,6 +4485,9 @@ class StubGenerator: public StubCodeGenerator { __ emit_int32(0x8f1bbcdc); __ emit_int32(0xca62c1d6); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3943,25 +4512,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } - - static const uint32_t round_consts[64] = { - 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, - 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, - 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, - 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, - 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, - 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, - 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, - 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, - 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, - 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, - 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, - 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, - 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, - 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, - 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, - 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, - }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); @@ -3987,7 +4545,7 @@ class StubGenerator: public StubCodeGenerator { // t1 == v7 // load 16 keys to v16..v31 - __ lea(rscratch1, ExternalAddress((address)round_consts)); + __ lea(rscratch1, ExternalAddress((address)_sha256_round_consts)); __ ld1(v16, v17, v18, v19, __ T4S, __ post(rscratch1, 64)); __ ld1(v20, v21, v22, v23, __ T4S, __ post(rscratch1, 64)); __ ld1(v24, v25, v26, v27, __ T4S, __ post(rscratch1, 64)); @@ -4048,6 +4606,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -4099,36 +4660,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } - - static const uint64_t round_consts[80] = { - 0x428A2F98D728AE22L, 0x7137449123EF65CDL, 0xB5C0FBCFEC4D3B2FL, - 0xE9B5DBA58189DBBCL, 0x3956C25BF348B538L, 0x59F111F1B605D019L, - 0x923F82A4AF194F9BL, 0xAB1C5ED5DA6D8118L, 0xD807AA98A3030242L, - 0x12835B0145706FBEL, 0x243185BE4EE4B28CL, 0x550C7DC3D5FFB4E2L, - 0x72BE5D74F27B896FL, 0x80DEB1FE3B1696B1L, 0x9BDC06A725C71235L, - 0xC19BF174CF692694L, 0xE49B69C19EF14AD2L, 0xEFBE4786384F25E3L, - 0x0FC19DC68B8CD5B5L, 0x240CA1CC77AC9C65L, 0x2DE92C6F592B0275L, - 0x4A7484AA6EA6E483L, 0x5CB0A9DCBD41FBD4L, 0x76F988DA831153B5L, - 0x983E5152EE66DFABL, 0xA831C66D2DB43210L, 0xB00327C898FB213FL, - 0xBF597FC7BEEF0EE4L, 0xC6E00BF33DA88FC2L, 0xD5A79147930AA725L, - 0x06CA6351E003826FL, 0x142929670A0E6E70L, 0x27B70A8546D22FFCL, - 0x2E1B21385C26C926L, 0x4D2C6DFC5AC42AEDL, 0x53380D139D95B3DFL, - 0x650A73548BAF63DEL, 0x766A0ABB3C77B2A8L, 0x81C2C92E47EDAEE6L, - 0x92722C851482353BL, 0xA2BFE8A14CF10364L, 0xA81A664BBC423001L, - 0xC24B8B70D0F89791L, 0xC76C51A30654BE30L, 0xD192E819D6EF5218L, - 0xD69906245565A910L, 0xF40E35855771202AL, 0x106AA07032BBD1B8L, - 0x19A4C116B8D2D0C8L, 0x1E376C085141AB53L, 0x2748774CDF8EEB99L, - 0x34B0BCB5E19B48A8L, 0x391C0CB3C5C95A63L, 0x4ED8AA4AE3418ACBL, - 0x5B9CCA4F7763E373L, 0x682E6FF3D6B2B8A3L, 0x748F82EE5DEFB2FCL, - 0x78A5636F43172F60L, 0x84C87814A1F0AB72L, 0x8CC702081A6439ECL, - 0x90BEFFFA23631E28L, 0xA4506CEBDE82BDE9L, 0xBEF9A3F7B2C67915L, - 0xC67178F2E372532BL, 0xCA273ECEEA26619CL, 0xD186B8C721C0C207L, - 0xEADA7DD6CDE0EB1EL, 0xF57D4F7FEE6ED178L, 0x06F067AA72176FBAL, - 0x0A637DC5A2C898A6L, 0x113F9804BEF90DAEL, 0x1B710B35131C471BL, - 0x28DB77F523047D84L, 0x32CAAB7B40C72493L, 0x3C9EBE0A15C9BEBCL, - 0x431D67C49C100D4CL, 0x4CC5D4BECB3E42B6L, 0x597F299CFC657E2AL, - 0x5FCB6FAB3AD6FAECL, 0x6C44198C4A475817L - }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); @@ -4151,7 +4690,7 @@ class StubGenerator: public StubCodeGenerator { __ ld1(v8, v9, v10, v11, __ T2D, state); // load first 4 round constants - __ lea(rscratch1, ExternalAddress((address)round_consts)); + __ lea(rscratch1, ExternalAddress((address)_sha512_round_consts)); __ ld1(v20, v21, v22, v23, __ T2D, __ post(rscratch1, 64)); __ BIND(sha512_loop); @@ -4236,6 +4775,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -4349,17 +4891,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } - - static const uint64_t round_consts[24] = { - 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL, - 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L, - 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL, - 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL, - 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L, - 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L, - 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L, - 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L - }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); @@ -4396,7 +4935,7 @@ class StubGenerator: public StubCodeGenerator { __ movw(rscratch2, 24); // load round_constants base - __ lea(rscratch1, ExternalAddress((address) round_consts)); + __ lea(rscratch1, ExternalAddress((address) _sha3_round_consts)); // load input __ ld1(v25, v26, v27, v28, __ T8B, __ post(buf, 32)); @@ -4488,6 +5027,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -4495,21 +5037,21 @@ class StubGenerator: public StubCodeGenerator { // c_rarg0 - long[] state0 // c_rarg1 - long[] state1 address generate_double_keccak() { - static const uint64_t round_consts[24] = { - 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL, - 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L, - 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL, - 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL, - 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L, - 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L, - 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L, - 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L - }; + StubId stub_id = StubId::stubgen_double_keccak_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } // Implements the double_keccak() method of the // sun.secyrity.provider.SHA3Parallel class __ align(CodeEntryAlignment); - StubCodeMark mark(this, "StubRoutines", "double_keccak"); + + StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -4546,7 +5088,7 @@ class StubGenerator: public StubCodeGenerator { __ movw(rscratch2, 24); // load round_constants base - __ lea(rscratch1, ExternalAddress((address) round_consts)); + __ lea(rscratch1, ExternalAddress((address) _double_keccak_round_consts)); __ BIND(rounds24_loop); __ subw(rscratch2, rscratch2, 1); @@ -4578,6 +5120,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -4611,9 +5156,18 @@ class StubGenerator: public StubCodeGenerator { // vectors write their first lane back to the keystream buffer, followed // by the second lane from all vectors and so on. address generate_chacha20Block_blockpar() { + StubId stub_id = StubId::stubgen_chacha20Block_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + Label L_twoRounds, L_cc20_const; __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_chacha20Block_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -4770,6 +5324,9 @@ class StubGenerator: public StubCodeGenerator { __ emit_int64(0x0605040702010003UL); __ emit_int64(0x0E0D0C0F0A09080BUL); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -5258,9 +5815,17 @@ class StubGenerator: public StubCodeGenerator { // coeffs (short[256]) = c_rarg0 // ntt_zetas (short[256]) = c_rarg1 address generate_kyberNtt() { + StubId stub_id = StubId::stubgen_kyberNtt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberNtt_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -5486,6 +6051,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -5496,9 +6064,17 @@ class StubGenerator: public StubCodeGenerator { // coeffs (short[256]) = c_rarg0 // ntt_zetas (short[256]) = c_rarg1 address generate_kyberInverseNtt() { + StubId stub_id = StubId::stubgen_kyberInverseNtt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberInverseNtt_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -5770,6 +6346,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -5783,9 +6362,17 @@ class StubGenerator: public StubCodeGenerator { // nttb (short[256]) = c_rarg2 // zetas (short[128]) = c_rarg3 address generate_kyberNttMult() { + StubId stub_id = StubId::stubgen_kyberNttMult_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberNttMult_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -5889,6 +6476,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -5900,9 +6490,17 @@ class StubGenerator: public StubCodeGenerator { // a (short[256]) = c_rarg1 // b (short[256]) = c_rarg2 address generate_kyberAddPoly_2() { + StubId stub_id = StubId::stubgen_kyberAddPoly_2_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberAddPoly_2_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -5973,6 +6571,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -5985,9 +6586,17 @@ class StubGenerator: public StubCodeGenerator { // b (short[256]) = c_rarg2 // c (short[256]) = c_rarg3 address generate_kyberAddPoly_3() { + StubId stub_id = StubId::stubgen_kyberAddPoly_3_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberAddPoly_3_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6072,6 +6681,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6088,10 +6700,19 @@ class StubGenerator: public StubCodeGenerator { // parsed (short[112 or 256]) = c_rarg2 // parsedLength (112 or 256) = c_rarg3 address generate_kyber12To16() { + StubId stub_id = StubId::stubgen_kyber12To16_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + Label L_F00, L_loop, L_end; __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyber12To16_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6288,6 +6909,9 @@ class StubGenerator: public StubCodeGenerator { __ emit_int64(0x0f000f000f000f00); __ emit_int64(0x0f000f000f000f00); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6297,9 +6921,17 @@ class StubGenerator: public StubCodeGenerator { // // coeffs (short[256]) = c_rarg0 address generate_kyberBarrettReduce() { + StubId stub_id = StubId::stubgen_kyberBarrettReduce_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_kyberBarrettReduce_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6381,6 +7013,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6544,9 +7179,17 @@ class StubGenerator: public StubCodeGenerator { // coeffs (int[256]) = c_rarg0 // zetas (int[256]) = c_rarg1 address generate_dilithiumAlmostNtt() { + StubId stub_id = StubId::stubgen_dilithiumAlmostNtt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_dilithiumAlmostNtt_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6659,6 +7302,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6751,9 +7397,17 @@ class StubGenerator: public StubCodeGenerator { // coeffs (int[256]) = c_rarg0 // zetas (int[256]) = c_rarg1 address generate_dilithiumAlmostInverseNtt() { + StubId stub_id = StubId::stubgen_dilithiumAlmostInverseNtt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_dilithiumAlmostInverseNtt_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6851,6 +7505,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6864,9 +7521,17 @@ class StubGenerator: public StubCodeGenerator { // poly1 (int[256]) = c_rarg1 // poly2 (int[256]) = c_rarg2 address generate_dilithiumNttMult() { - - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_dilithiumNttMult_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6917,6 +7582,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6928,9 +7596,17 @@ class StubGenerator: public StubCodeGenerator { // coeffs (int[256]) = c_rarg0 // constant (int) = c_rarg1 address generate_dilithiumMontMulByConstant() { + StubId stub_id = StubId::stubgen_dilithiumMontMulByConstant_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_dilithiumMontMulByConstant_id; StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); @@ -6978,6 +7654,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -6992,9 +7671,17 @@ class StubGenerator: public StubCodeGenerator { // twoGamma2 (int) = c_rarg3 // multiplier (int) = c_rarg4 address generate_dilithiumDecomposePoly() { + StubId stub_id = StubId::stubgen_dilithiumDecomposePoly_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_dilithiumDecomposePoly_id; StubCodeMark mark(this, stub_id); address start = __ pc(); Label L_loop; @@ -7136,6 +7823,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(r0, zr); // return 0 __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7275,17 +7965,14 @@ class StubGenerator: public StubCodeGenerator { default: ShouldNotReachHere(); } - - static const uint64_t round_consts[24] = { - 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL, - 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L, - 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL, - 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL, - 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L, - 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L, - 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L, - 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L - }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -7441,7 +8128,7 @@ class StubGenerator: public StubCodeGenerator { __ fmovs(v1, 1.0); // exact representation __ str(buf, Address(sp, 16)); - __ lea(tmp3, ExternalAddress((address) round_consts)); + __ lea(tmp3, ExternalAddress((address) _sha3_round_consts)); __ BIND(loop_body); keccak_round_gpr(can_use_fp, can_use_r18, tmp3, @@ -7496,6 +8183,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7512,9 +8202,17 @@ class StubGenerator: public StubCodeGenerator { */ address generate_updateBytesCRC32() { assert(UseCRC32Intrinsics, "what are we doing here?"); + StubId stub_id = StubId::stubgen_updateBytesCRC32_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_updateBytesCRC32_id; StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7537,6 +8235,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7554,9 +8255,17 @@ class StubGenerator: public StubCodeGenerator { */ address generate_updateBytesCRC32C() { assert(UseCRC32CIntrinsics, "what are we doing here?"); + StubId stub_id = StubId::stubgen_updateBytesCRC32C_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_updateBytesCRC32C_id; StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7579,6 +8288,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7594,8 +8306,17 @@ class StubGenerator: public StubCodeGenerator { * c_rarg0 - int adler result */ address generate_updateBytesAdler32() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_updateBytesAdler32_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7765,6 +8486,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7816,8 +8540,17 @@ class StubGenerator: public StubCodeGenerator { * c_rarg4 - z address */ address generate_multiplyToLen() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_multiplyToLen_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7842,6 +8575,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7849,8 +8585,17 @@ class StubGenerator: public StubCodeGenerator { // squareToLen algorithm for sizes 1..127 described in java code works // faster than multiply_to_len on some CPUs and slower on others, but // multiply_to_len shows a bit better overall results - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_squareToLen_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7879,12 +8624,25 @@ class StubGenerator: public StubCodeGenerator { __ pop(spilled_regs, sp); __ leave(); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address generate_mulAdd() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_mulAdd_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -7901,6 +8659,9 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -7914,8 +8675,17 @@ class StubGenerator: public StubCodeGenerator { // c_rarg4 - numIter // address generate_bigIntegerRightShift() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_bigIntegerRightShiftWorker_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -8024,6 +8794,9 @@ class StubGenerator: public StubCodeGenerator { __ BIND(Exit); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -8039,6 +8812,15 @@ class StubGenerator: public StubCodeGenerator { address generate_bigIntegerLeftShift() { __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_bigIntegerLeftShiftWorker_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -8135,10 +8917,28 @@ class StubGenerator: public StubCodeGenerator { __ BIND(Exit); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address generate_count_positives(address &count_positives_long) { + StubId stub_id = StubId::stubgen_count_positives_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + // We have an extra entry for count_positives_long. + assert(entry_count == 2, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == 1, + "unexpected extra entry count %d", entries.length()); + count_positives_long = entries.at(0); + return start; + } + const u1 large_loop_size = 64; const uint64_t UPPER_BIT_MASK=0x8080808080808080; int dcache_line = VM_Version::dcache_line_size(); @@ -8147,7 +8947,6 @@ class StubGenerator: public StubCodeGenerator { __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_count_positives_id; StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -8190,6 +8989,7 @@ class StubGenerator: public StubCodeGenerator { const RegSet spilled_regs = RegSet::range(tmp1, tmp5) + tmp6; count_positives_long = __ pc(); // 2nd entry point + entries.append(count_positives_long); __ enter(); @@ -8304,6 +9104,9 @@ class StubGenerator: public StubCodeGenerator { __ sub(result, result, len); __ ret(lr); + // record the stub entry and end plus the extra entry + store_archive_data(stub_id, entry, __ pc(), &entries); + return entry; } @@ -8394,6 +9197,16 @@ class StubGenerator: public StubCodeGenerator { // r3-r5 are reserved temporary registers // Clobbers: v0-v7 when UseSIMDForArrayEquals, rscratch1, rscratch2 address generate_large_array_equals() { + StubId stub_id = StubId::stubgen_large_array_equals_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1, tmp2 = rscratch2, tmp3 = r3, tmp4 = r4, tmp5 = r5, tmp6 = r11, tmp7 = r12, tmp8 = r13; @@ -8409,7 +9222,6 @@ class StubGenerator: public StubCodeGenerator { __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_large_array_equals_id; StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -8484,6 +9296,10 @@ class StubGenerator: public StubCodeGenerator { __ bind(NOT_EQUAL_NO_POP); __ leave(); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -8492,6 +9308,36 @@ class StubGenerator: public StubCodeGenerator { // cnt = r2 - elements count // Clobbers: v0-v13, rscratch1, rscratch2 address generate_large_arrays_hashcode(BasicType eltype) { + StubId stub_id; + switch (eltype) { + case T_BOOLEAN: + stub_id = StubId::stubgen_large_arrays_hashcode_boolean_id; + break; + case T_BYTE: + stub_id = StubId::stubgen_large_arrays_hashcode_byte_id; + break; + case T_CHAR: + stub_id = StubId::stubgen_large_arrays_hashcode_char_id; + break; + case T_SHORT: + stub_id = StubId::stubgen_large_arrays_hashcode_short_id; + break; + case T_INT: + stub_id = StubId::stubgen_large_arrays_hashcode_int_id; + break; + default: + stub_id = StubId::NO_STUBID; + ShouldNotReachHere(); + }; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + const Register result = r0, ary = r1, cnt = r2; const FloatRegister vdata0 = v3, vdata1 = v2, vdata2 = v1, vdata3 = v0; const FloatRegister vmul0 = v4, vmul1 = v5, vmul2 = v6, vmul3 = v7; @@ -8535,28 +9381,6 @@ class StubGenerator: public StubCodeGenerator { __ align(CodeEntryAlignment); - StubId stub_id; - switch (eltype) { - case T_BOOLEAN: - stub_id = StubId::stubgen_large_arrays_hashcode_boolean_id; - break; - case T_BYTE: - stub_id = StubId::stubgen_large_arrays_hashcode_byte_id; - break; - case T_CHAR: - stub_id = StubId::stubgen_large_arrays_hashcode_char_id; - break; - case T_SHORT: - stub_id = StubId::stubgen_large_arrays_hashcode_short_id; - break; - case T_INT: - stub_id = StubId::stubgen_large_arrays_hashcode_int_id; - break; - default: - stub_id = StubId::NO_STUBID; - ShouldNotReachHere(); - }; - StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -8791,12 +9615,24 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } address generate_dsin_dcos(bool isCos) { - __ align(CodeEntryAlignment); StubId stub_id = (isCos ? StubId::stubgen_dcos_id : StubId::stubgen_dsin_id); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ generate_dsin_dcos(isCos, (address)StubRoutines::aarch64::_npio2_hw, @@ -8804,6 +9640,10 @@ class StubGenerator: public StubCodeGenerator { (address)StubRoutines::aarch64::_pio2, (address)StubRoutines::aarch64::_dsin_coef, (address)StubRoutines::aarch64::_dcos_coef); + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -8847,8 +9687,17 @@ class StubGenerator: public StubCodeGenerator { // r10 = tmp1 // r11 = tmp2 address generate_compare_long_string_different_encoding(bool isLU) { - __ align(CodeEntryAlignment); StubId stub_id = (isLU ? StubId::stubgen_compare_long_string_LU_id : StubId::stubgen_compare_long_string_UL_id); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); Label SMALL_LOOP, TAIL, TAIL_LOAD_16, LOAD_LAST, DIFF1, DIFF2, @@ -8950,20 +9799,37 @@ class StubGenerator: public StubCodeGenerator { __ subw(result, tmp1, rscratch1); __ bind(DONE); __ ret(lr); - return entry; + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + + return entry; } // r0 = input (float16) // v0 = result (float) // v1 = temporary float register address generate_float16ToFloat() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_hf2f_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); BLOCK_COMMENT("Entry:"); __ flt16_to_flt(v0, r0, v1); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -8971,19 +9837,39 @@ class StubGenerator: public StubCodeGenerator { // r0 = result (float16) // v1 = temporary float register address generate_floatToFloat16() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_f2hf_id; + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); BLOCK_COMMENT("Entry:"); __ flt_to_flt16(r0, v0, v1); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } address generate_method_entry_barrier() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_method_entry_barrier_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); Label deoptimize_label; @@ -9037,6 +9923,9 @@ class StubGenerator: public StubCodeGenerator { __ mov(sp, rscratch1); __ br(rscratch2); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -9048,8 +9937,18 @@ class StubGenerator: public StubCodeGenerator { // r10 = tmp1 // r11 = tmp2 address generate_compare_long_string_same_encoding(bool isLL) { - __ align(CodeEntryAlignment); StubId stub_id = (isLL ? StubId::stubgen_compare_long_string_LL_id : StubId::stubgen_compare_long_string_UU_id); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); + StubCodeMark mark(this, stub_id); address entry = __ pc(); Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4, @@ -9157,6 +10056,10 @@ class StubGenerator: public StubCodeGenerator { __ bind(LENGTH_DIFF); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -9188,8 +10091,17 @@ class StubGenerator: public StubCodeGenerator { case UU: stub_id = StubId::stubgen_compare_long_string_UU_id; break; default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); + StubCodeMark mark(this, stub_id); address entry = __ pc(); Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4, tmp1 = r10, tmp2 = r11; @@ -9224,8 +10136,6 @@ class StubGenerator: public StubCodeGenerator { ShouldNotReachHere(); \ } - StubCodeMark mark(this, stub_id); - __ mov(idx, 0); __ sve_whilelt(pgtmp1, mode == LL ? __ B : __ H, idx, cnt); @@ -9269,6 +10179,10 @@ class StubGenerator: public StubCodeGenerator { __ bind(DONE); __ ret(lr); #undef LOAD_PAIR + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -9330,6 +10244,15 @@ class StubGenerator: public StubCodeGenerator { stub_id = StubId::stubgen_string_indexof_linear_uu_id; } } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); @@ -9598,6 +10521,10 @@ class StubGenerator: public StubCodeGenerator { __ BIND(DONE); __ pop(spilled_regs, sp); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -9628,8 +10555,17 @@ class StubGenerator: public StubCodeGenerator { // v1 = loaded 8 bytes // Clobbers: r0, r1, r3, rscratch1, rflags, v0-v6 address generate_large_byte_array_inflate() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_large_byte_array_inflate_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address entry = __ pc(); Label LOOP, LOOP_START, LOOP_PRFM, LOOP_PRFM_START, DONE; @@ -9668,6 +10604,10 @@ class StubGenerator: public StubCodeGenerator { __ br(__ GE, LOOP); __ bind(DONE); __ ret(lr); + + // record the stub entry and end + store_archive_data(stub_id, entry, __ pc()); + return entry; } @@ -9683,7 +10623,7 @@ class StubGenerator: public StubCodeGenerator { * Output: * Updated state at c_rarg0 */ - address generate_ghash_processBlocks() { + address generate_ghash_processBlocks_small() { // Bafflingly, GCM uses little-endian for the byte order, but // big-endian for the bit order. For example, the polynomial 1 is // represented as the 16-byte string 80 00 00 00 | 12 bytes of 00. @@ -9695,7 +10635,16 @@ class StubGenerator: public StubCodeGenerator { // that) and keep the data in little-endian bit order through the // calculation, bit-reversing the inputs and outputs. - StubId stub_id = StubId::stubgen_ghash_processBlocks_id; + StubId stub_id = StubId::stubgen_ghash_processBlocks_small_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); Label polynomial; // local data generated at end of stub __ align(CodeEntryAlignment); @@ -9759,16 +10708,26 @@ class StubGenerator: public StubCodeGenerator { // 128-bit vector __ emit_int64(0x87); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } - address generate_ghash_processBlocks_wide() { - address small = generate_ghash_processBlocks(); + address generate_ghash_processBlocks(address small) { + StubId stub_id = StubId::stubgen_ghash_processBlocks_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } - StubId stub_id = StubId::stubgen_ghash_processBlocks_wide_id; - StubCodeMark mark(this, stub_id); Label polynomial; // local data generated after stub __ align(CodeEntryAlignment); + StubCodeMark mark(this, stub_id); address start = __ pc(); Register state = c_rarg0; @@ -9811,8 +10770,10 @@ class StubGenerator: public StubCodeGenerator { // 128-bit vector __ emit_int64(0x87); - return start; + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } void generate_base64_encode_simdround(Register src, Register dst, @@ -9863,24 +10824,17 @@ class StubGenerator: public StubCodeGenerator { */ address generate_base64_encodeBlock() { - static const char toBase64[64] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', - 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', - 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/' - }; - - static const char toBase64URL[64] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', - 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', - 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_' - }; + StubId stub_id = StubId::stubgen_base64_encodeBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_base64_encodeBlock_id; StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -9902,9 +10856,9 @@ class StubGenerator: public StubCodeGenerator { __ sub(length, send, soff); // load the codec base address - __ lea(codec, ExternalAddress((address) toBase64)); + __ lea(codec, ExternalAddress((address) _encodeBlock_toBase64)); __ cbz(isURL, ProcessData); - __ lea(codec, ExternalAddress((address) toBase64URL)); + __ lea(codec, ExternalAddress((address) _encodeBlock_toBase64URL)); __ BIND(ProcessData); @@ -9957,6 +10911,9 @@ class StubGenerator: public StubCodeGenerator { __ BIND(Exit); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10078,78 +11035,17 @@ class StubGenerator: public StubCodeGenerator { // on http://0x80.pl/articles/base64-simd-neon.html#encoding-quadwords, in section // titled "Base64 decoding". - // Non-SIMD lookup tables are mostly dumped from fromBase64 array used in java.util.Base64, - // except the trailing character '=' is also treated illegal value in this intrinsic. That - // is java.util.Base64.fromBase64['='] = -2, while fromBase(URL)64ForNoSIMD['='] = 255 here. - static const uint8_t fromBase64ForNoSIMD[256] = { - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u, - 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, - 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 255u, - 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u, - 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - }; - - static const uint8_t fromBase64URLForNoSIMD[256] = { - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, - 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, - 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 63u, - 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u, - 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - }; - - // A legal value of base64 code is in range [0, 127]. We need two lookups - // with tbl/tbx and combine them to get the decode data. The 1st table vector - // lookup use tbl, out of range indices are set to 0 in destination. The 2nd - // table vector lookup use tbx, out of range indices are unchanged in - // destination. Input [64..126] is mapped to index [65, 127] in second lookup. - // The value of index 64 is set to 0, so that we know that we already get the - // decoded data with the 1st lookup. - static const uint8_t fromBase64ForSIMD[128] = { - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u, - 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, - 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, - 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, - 255u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, - 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, - }; - - static const uint8_t fromBase64URLForSIMD[128] = { - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, - 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, - 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u, - 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, - 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, - 63u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, - 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, - }; + StubId stub_id = StubId::stubgen_base64_decodeBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_base64_decodeBlock_id; StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -10178,9 +11074,9 @@ class StubGenerator: public StubCodeGenerator { __ sub(length, send, soff); __ bfm(length, zr, 0, 1); - __ lea(nosimd_codec, ExternalAddress((address) fromBase64ForNoSIMD)); + __ lea(nosimd_codec, ExternalAddress((address) _decodeBlock_fromBase64ForNoSIMD)); __ cbz(isURL, ProcessData); - __ lea(nosimd_codec, ExternalAddress((address) fromBase64URLForNoSIMD)); + __ lea(nosimd_codec, ExternalAddress((address) _decodeBlock_fromBase64URLForNoSIMD)); __ BIND(ProcessData); __ mov(rscratch1, length); @@ -10225,9 +11121,9 @@ class StubGenerator: public StubCodeGenerator { __ cbzw(rscratch1, Exit); __ sub(length, length, 80); - __ lea(simd_codec, ExternalAddress((address) fromBase64ForSIMD)); + __ lea(simd_codec, ExternalAddress((address) _decodeBlock_fromBase64ForSIMD)); __ cbz(isURL, SIMDEnter); - __ lea(simd_codec, ExternalAddress((address) fromBase64URLForSIMD)); + __ lea(simd_codec, ExternalAddress((address) _decodeBlock_fromBase64URLForSIMD)); __ BIND(SIMDEnter); __ ld1(v0, v1, v2, v3, __ T16B, __ post(simd_codec, 64)); @@ -10260,24 +11156,55 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } // Support for spin waits. address generate_spin_wait() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_spin_wait_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ spin_wait(); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } void generate_lookup_secondary_supers_table_stub() { StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == Klass::SECONDARY_SUPERS_TABLE_SIZE, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == Klass::SECONDARY_SUPERS_TABLE_SIZE - 1, + "unexpected extra entry count %d", entries.length()); + StubRoutines::_lookup_secondary_supers_table_stubs[0] = start; + for (int slot = 1; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) { + StubRoutines::_lookup_secondary_supers_table_stubs[0] = entries.at(slot - 1); + } + return; + } + StubCodeMark mark(this, stub_id); const Register @@ -10291,8 +11218,15 @@ class StubGenerator: public StubCodeGenerator { const FloatRegister vtemp = v0; + address start; for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) { - StubRoutines::_lookup_secondary_supers_table_stubs[slot] = __ pc(); + address next_entry = __ pc(); + StubRoutines::_lookup_secondary_supers_table_stubs[slot] = next_entry; + if (slot == 0) { + start = next_entry; + } else { + entries.append(next_entry); + } Label L_success; __ enter(); __ lookup_secondary_supers_table_const(r_sub_klass, r_super_klass, @@ -10302,11 +11236,22 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); } + // record the stub entry and end plus all the auxiliary entries + store_archive_data(stub_id, start, __ pc(), &entries); } // Slow path implementation for UseSecondarySupersTable. address generate_lookup_secondary_supers_table_slow_path_stub() { StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_slow_path_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -10321,6 +11266,9 @@ class StubGenerator: public StubCodeGenerator { __ lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, result); __ ret(lr); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10460,14 +11408,45 @@ class StubGenerator: public StubCodeGenerator { if (! UseLSE) { return; } - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_atomic_entry_points_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == entry_count - 1, + "unexpected extra entry count %d", entries.length()); + aarch64_atomic_fetch_add_4_impl = (aarch64_atomic_stub_t)start; + int idx = 0; + aarch64_atomic_fetch_add_8_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_fetch_add_4_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_fetch_add_8_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_xchg_4_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_xchg_8_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_1_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_4_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_8_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_1_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_4_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_8_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_4_release_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_8_release_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_4_seq_cst_impl = (aarch64_atomic_stub_t)entries.at(idx++); + aarch64_atomic_cmpxchg_8_seq_cst_impl = (aarch64_atomic_stub_t)entries.at(idx++); + assert(idx == entries.length(), "sanity!"); + return; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address first_entry = __ pc(); - + address end; + { // ADD, memory_order_conservative AtomicStubMark mark_fetch_add_4(_masm, &aarch64_atomic_fetch_add_4_impl); gen_ldadd_entry(Assembler::word, memory_order_conservative); + AtomicStubMark mark_fetch_add_8(_masm, &aarch64_atomic_fetch_add_8_impl); gen_ldadd_entry(Assembler::xword, memory_order_conservative); @@ -10475,6 +11454,7 @@ class StubGenerator: public StubCodeGenerator { AtomicStubMark mark_fetch_add_4_relaxed (_masm, &aarch64_atomic_fetch_add_4_relaxed_impl); gen_ldadd_entry(MacroAssembler::word, memory_order_relaxed); + AtomicStubMark mark_fetch_add_8_relaxed (_masm, &aarch64_atomic_fetch_add_8_relaxed_impl); gen_ldadd_entry(MacroAssembler::xword, memory_order_relaxed); @@ -10482,14 +11462,17 @@ class StubGenerator: public StubCodeGenerator { // XCHG, memory_order_conservative AtomicStubMark mark_xchg_4(_masm, &aarch64_atomic_xchg_4_impl); gen_swpal_entry(Assembler::word); - AtomicStubMark mark_xchg_8_impl(_masm, &aarch64_atomic_xchg_8_impl); + + AtomicStubMark mark_xchg_8(_masm, &aarch64_atomic_xchg_8_impl); gen_swpal_entry(Assembler::xword); // CAS, memory_order_conservative AtomicStubMark mark_cmpxchg_1(_masm, &aarch64_atomic_cmpxchg_1_impl); gen_cas_entry(MacroAssembler::byte, memory_order_conservative); + AtomicStubMark mark_cmpxchg_4(_masm, &aarch64_atomic_cmpxchg_4_impl); gen_cas_entry(MacroAssembler::word, memory_order_conservative); + AtomicStubMark mark_cmpxchg_8(_masm, &aarch64_atomic_cmpxchg_8_impl); gen_cas_entry(MacroAssembler::xword, memory_order_conservative); @@ -10497,9 +11480,11 @@ class StubGenerator: public StubCodeGenerator { AtomicStubMark mark_cmpxchg_1_relaxed (_masm, &aarch64_atomic_cmpxchg_1_relaxed_impl); gen_cas_entry(MacroAssembler::byte, memory_order_relaxed); + AtomicStubMark mark_cmpxchg_4_relaxed (_masm, &aarch64_atomic_cmpxchg_4_relaxed_impl); gen_cas_entry(MacroAssembler::word, memory_order_relaxed); + AtomicStubMark mark_cmpxchg_8_relaxed (_masm, &aarch64_atomic_cmpxchg_8_relaxed_impl); gen_cas_entry(MacroAssembler::xword, memory_order_relaxed); @@ -10507,6 +11492,7 @@ class StubGenerator: public StubCodeGenerator { AtomicStubMark mark_cmpxchg_4_release (_masm, &aarch64_atomic_cmpxchg_4_release_impl); gen_cas_entry(MacroAssembler::word, memory_order_release); + AtomicStubMark mark_cmpxchg_8_release (_masm, &aarch64_atomic_cmpxchg_8_release_impl); gen_cas_entry(MacroAssembler::xword, memory_order_release); @@ -10514,11 +11500,41 @@ class StubGenerator: public StubCodeGenerator { AtomicStubMark mark_cmpxchg_4_seq_cst (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl); gen_cas_entry(MacroAssembler::word, memory_order_seq_cst); + AtomicStubMark mark_cmpxchg_8_seq_cst (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl); gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst); - ICache::invalidate_range(first_entry, __ pc() - first_entry); + end = __ pc(); + + ICache::invalidate_range(first_entry, end - first_entry); + // exit block to force update of AtomicStubMark targets + } + + assert(first_entry == (address)aarch64_atomic_fetch_add_4_impl, + "atomic stub should be at start of buffer"); + // record the stub start and end plus all the entries saved by the + // AtomicStubMark destructor + entries.append((address)aarch64_atomic_fetch_add_8_impl); + entries.append((address)aarch64_atomic_fetch_add_4_relaxed_impl); + entries.append((address)aarch64_atomic_fetch_add_8_relaxed_impl); + entries.append((address)aarch64_atomic_xchg_4_impl); + entries.append((address)aarch64_atomic_xchg_8_impl); + entries.append((address)aarch64_atomic_cmpxchg_1_impl); + entries.append((address)aarch64_atomic_cmpxchg_4_impl); + entries.append((address)aarch64_atomic_cmpxchg_8_impl); + entries.append((address)aarch64_atomic_cmpxchg_1_relaxed_impl); + entries.append((address)aarch64_atomic_cmpxchg_4_relaxed_impl); + entries.append((address)aarch64_atomic_cmpxchg_8_relaxed_impl); + entries.append((address)aarch64_atomic_cmpxchg_4_release_impl); + entries.append((address)aarch64_atomic_cmpxchg_8_release_impl); + entries.append((address)aarch64_atomic_cmpxchg_4_seq_cst_impl); + entries.append((address)aarch64_atomic_cmpxchg_8_seq_cst_impl); + + assert(entries.length() == entry_count - 1, + "unexpected extra entry count %d", entries.length()); + + store_archive_data(stub_id, first_entry, end, &entries); } #endif // LINUX @@ -10622,9 +11638,22 @@ class StubGenerator: public StubCodeGenerator { if (!Continuations::enabled()) return nullptr; StubId stub_id = StubId::stubgen_cont_thaw_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); generate_cont_thaw(Continuation::thaw_top); + + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10633,11 +11662,23 @@ class StubGenerator: public StubCodeGenerator { // TODO: will probably need multiple return barriers depending on return type StubId stub_id = StubId::stubgen_cont_returnBarrier_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); generate_cont_thaw(Continuation::thaw_return_barrier); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10645,17 +11686,38 @@ class StubGenerator: public StubCodeGenerator { if (!Continuations::enabled()) return nullptr; StubId stub_id = StubId::stubgen_cont_returnBarrierExc_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); generate_cont_thaw(Continuation::thaw_return_barrier_exception); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } address generate_cont_preempt_stub() { if (!Continuations::enabled()) return nullptr; StubId stub_id = StubId::stubgen_cont_preempt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -10682,6 +11744,9 @@ class StubGenerator: public StubCodeGenerator { __ ldr(rscratch1, Address(rscratch1)); __ br(rscratch1); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10737,8 +11802,17 @@ class StubGenerator: public StubCodeGenerator { // computation. address generate_poly1305_processBlocks() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_poly1305_processBlocks_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); Label here; @@ -10848,12 +11922,24 @@ class StubGenerator: public StubCodeGenerator { __ leave(); __ ret(lr); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } // exception handler for upcall stubs address generate_upcall_stub_exception_handler() { StubId stub_id = StubId::stubgen_upcall_stub_exception_handler_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -10864,6 +11950,9 @@ class StubGenerator: public StubCodeGenerator { __ blr(rscratch1); __ should_not_reach_here(); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -10872,6 +11961,15 @@ class StubGenerator: public StubCodeGenerator { // rmethod = result address generate_upcall_stub_load_target() { StubId stub_id = StubId::stubgen_upcall_stub_load_target_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -10887,6 +11985,9 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); + // record the stub start and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -11915,7 +13016,8 @@ class StubGenerator: public StubCodeGenerator { } if (UseGHASHIntrinsics) { // StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks(); - StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks_wide(); + StubRoutines::aarch64::_ghash_processBlocks_small = generate_ghash_processBlocks_small(); + StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks(StubRoutines::aarch64::_ghash_processBlocks_small); } if (UseAESIntrinsics && UseGHASHIntrinsics) { StubRoutines::_galoisCounterMode_AESCrypt = generate_galoisCounterMode_AESCrypt(); @@ -11962,7 +13064,7 @@ class StubGenerator: public StubCodeGenerator { } public: - StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) { + StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) { switch(blob_id) { case BlobId::stubgen_preuniverse_id: generate_preuniverse_stubs(); @@ -11984,10 +13086,36 @@ class StubGenerator: public StubCodeGenerator { break; }; } + + static void init_AOTAddressTable(GrowableArray
& external_addresses) { + // external data defined in this file +#define ADD(addr) external_addresses.append((address)addr); + ADD(_sha256_round_consts); + ADD(_sha512_round_consts); + ADD(_sha3_round_consts); + ADD(_double_keccak_round_consts); + ADD(_encodeBlock_toBase64); + ADD(_encodeBlock_toBase64URL); + ADD(_decodeBlock_fromBase64ForNoSIMD); + ADD(_decodeBlock_fromBase64URLForNoSIMD); + ADD(_decodeBlock_fromBase64ForSIMD); + ADD(_decodeBlock_fromBase64URLForSIMD); +#undef ADD + } }; // end class declaration -void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) { - StubGenerator g(code, blob_id); +void StubGenerator_AOTAddressTable_init() { + ResourceMark rm; + GrowableArray
external_addresses; + // publish static addresses referred to by aarch64 generator + StubGenerator::init_AOTAddressTable(external_addresses); + // publish external data addresses defined in nested aarch64 class + StubRoutines::aarch64::init_AOTAddressTable(external_addresses); + AOTCodeCache::publish_external_addresses(external_addresses); +} + +void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { + StubGenerator g(code, blob_id, stub_data); } diff --git a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp index 88993818b47ee..73f0797cb7aca 100644 --- a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp @@ -413,3 +413,19 @@ ATTRIBUTE_ALIGNED(64) jdouble StubRoutines::aarch64::_pio2[] = { 2.73370053816464559624e-44, // 0x36E3822280000000 2.16741683877804819444e-51, // 0x3569F31D00000000 }; + +#define ADD(addr) external_addresses.append((address)addr); + +void StubRoutines::aarch64::init_AOTAddressTable(GrowableArray
& external_addresses) { + ADD(_kyberConsts); + ADD(_dilithiumConsts); + ADD(_crc_table); + ADD(_adler_table); + ADD(_npio2_hw); + ADD(_dsin_coef); + ADD(_dcos_coef); + ADD(_two_over_pi); + ADD(_pio2); +} + +#undef ADD diff --git a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp index c35371e10833d..89c72a6a5ea11 100644 --- a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp @@ -110,6 +110,8 @@ class aarch64 { _completed = true; } + static void init_AOTAddressTable(GrowableArray
& external_addresses); + private: static uint16_t _kyberConsts[]; static uint32_t _dilithiumConsts[]; diff --git a/src/hotspot/cpu/arm/stubGenerator_arm.cpp b/src/hotspot/cpu/arm/stubGenerator_arm.cpp index a36ad3a0c4701..16155203278e2 100644 --- a/src/hotspot/cpu/arm/stubGenerator_arm.cpp +++ b/src/hotspot/cpu/arm/stubGenerator_arm.cpp @@ -3211,7 +3211,7 @@ class StubGenerator: public StubCodeGenerator { } public: - StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) { + StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) { switch(blob_id) { case BlobId::stubgen_preuniverse_id: generate_preuniverse_stubs(); @@ -3235,8 +3235,12 @@ class StubGenerator: public StubCodeGenerator { } }; // end class declaration -void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) { - StubGenerator g(code, blob_id); +// nothing to do for arm +void StubGenerator_AOTAddressTable_init() { +} + +void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { + StubGenerator g(code, blob_id, stub_data); } // implementation of internal development flag diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp index 948092bbb9aad..2156f742c3a10 100644 --- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp +++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp @@ -5095,7 +5095,7 @@ void generate_lookup_secondary_supers_table_stub() { } public: - StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) { + StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) : StubCodeGenerator(code, blob_id, stub_data) { switch(blob_id) { case BlobId::stubgen_preuniverse_id: generate_preuniverse_stubs(); @@ -5119,7 +5119,11 @@ void generate_lookup_secondary_supers_table_stub() { } }; -void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) { - StubGenerator g(code, blob_id); +// nothing to do for ppc +void StubGenerator_AOTAddressTable_init() { +} + +void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) { + StubGenerator g(code, blob_id, stub_data); } diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp index e5eb15cb8e4ef..e8bd832612a4d 100644 --- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp +++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp @@ -6882,7 +6882,7 @@ static const int64_t right_3_bits = right_n_bits(3); } public: - StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) { + StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) { switch(blob_id) { case BlobId::stubgen_preuniverse_id: generate_preuniverse_stubs(); @@ -6906,6 +6906,10 @@ static const int64_t right_3_bits = right_n_bits(3); } }; // end class declaration -void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) { - StubGenerator g(code, blob_id); +// nothing to do for riscv +void StubGenerator_AOTAddressTable_init() { +} + +void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { + StubGenerator g(code, blob_id, stub_data); } diff --git a/src/hotspot/cpu/s390/stubGenerator_s390.cpp b/src/hotspot/cpu/s390/stubGenerator_s390.cpp index 2aa365be99906..56ba3a85cbae4 100644 --- a/src/hotspot/cpu/s390/stubGenerator_s390.cpp +++ b/src/hotspot/cpu/s390/stubGenerator_s390.cpp @@ -3422,7 +3422,7 @@ class StubGenerator: public StubCodeGenerator { } public: - StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) { + StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) { switch(blob_id) { case BlobId::stubgen_preuniverse_id: generate_preuniverse_stubs(); @@ -3479,6 +3479,10 @@ class StubGenerator: public StubCodeGenerator { }; -void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) { - StubGenerator g(code, blob_id); +// nothing to do for s390 +void StubGenerator_AOTAddressTable_init() { +} + +void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { + StubGenerator g(code, blob_id, stub_data); } diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index edeb0baea0e95..fcdf55324e8ad 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -69,6 +69,15 @@ static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000)); static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000)); +// publish external addresses defined in this file +void LIR_Assembler::init_AOTAddressTable(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(float_signmask_pool); + ADD(double_signmask_pool); + ADD(float_signflip_pool); + ADD(double_signflip_pool); +#undef ADD +} NEEDS_CLEANUP // remove this definitions ? const Register SYNC_header = rax; // synchronization header diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp index 8524dc90276f0..71a137a1e901a 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp @@ -58,4 +58,5 @@ void store_parameter(jobject c, int offset_from_esp_in_words); void store_parameter(Metadata* c, int offset_from_esp_in_words); + void static init_AOTAddressTable(GrowableArray
& external_addresses); #endif // CPU_X86_C1_LIRASSEMBLER_X86_HPP diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index 44f1a35d443ab..fb033732ef6b6 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -5574,7 +5574,7 @@ void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) { void MacroAssembler::reinit_heapbase() { if (UseCompressedOops) { - if (Universe::heap() != nullptr) { + if (Universe::heap() != nullptr && !AOTCodeCache::is_on_for_dump()) { if (CompressedOops::base() == nullptr) { MacroAssembler::xorptr(r12_heapbase, r12_heapbase); } else { diff --git a/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp b/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp index 9f0232075cd6a..822400062fdd1 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp @@ -792,6 +792,9 @@ enum { movl(h, Address(CTX, 4*7)); pshuffle_byte_flip_mask_addr = pshuffle_byte_flip_mask; + assert(pshuffle_byte_flip_mask_addr + 32 == StubRoutines::x86::pshuffle_byte_flip_mask_00ba_addr(), "sanity"); + assert(pshuffle_byte_flip_mask_addr + 64 == StubRoutines::x86::pshuffle_byte_flip_mask_dc00_addr(), "sanity"); + vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // [PSHUFFLE_BYTE_FLIP_MASK wrt rip] vmovdqu(SHUF_00BA, ExternalAddress(pshuffle_byte_flip_mask_addr + 32)); // [_SHUF_00BA wrt rip] vmovdqu(SHUF_DC00, ExternalAddress(pshuffle_byte_flip_mask_addr + 64)); // [_SHUF_DC00 wrt rip] @@ -955,6 +958,9 @@ bind(only_one_block); pshuffle_byte_flip_mask_addr = pshuffle_byte_flip_mask; + assert(pshuffle_byte_flip_mask_addr + 32 == StubRoutines::x86::pshuffle_byte_flip_mask_00ba_addr(), "sanity"); + assert(pshuffle_byte_flip_mask_addr + 64 == StubRoutines::x86::pshuffle_byte_flip_mask_dc00_addr(), "sanity"); + vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // [PSHUFFLE_BYTE_FLIP_MASK wrt rip] vmovdqu(SHUF_00BA, ExternalAddress(pshuffle_byte_flip_mask_addr + 32)); // [_SHUF_00BA wrt rip] vmovdqu(SHUF_DC00, ExternalAddress(pshuffle_byte_flip_mask_addr + 64)); // [_SHUF_DC00 wrt rip] @@ -1347,7 +1353,9 @@ void MacroAssembler::sha512_AVX2(XMMRegister msg, XMMRegister state0, XMMRegiste movq(h, Address(CTX, 8 * 7)); pshuffle_byte_flip_mask_addr = pshuffle_byte_flip_mask_sha512; - vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // PSHUFFLE_BYTE_FLIP_MASK wrt rip + assert(pshuffle_byte_flip_mask_addr + 32 == StubRoutines::x86::pshuffle_byte_flip_mask_ymm_lo_addr_sha512(), "sanity"); + + vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // PSHUFFLE_BYTE_FLIP_MASK wrt rip vmovdqu(YMM_MASK_LO, ExternalAddress(pshuffle_byte_flip_mask_addr + 32)); movq(g, Address(CTX, 8 * 6)); diff --git a/src/hotspot/cpu/x86/stubDeclarations_x86.hpp b/src/hotspot/cpu/x86/stubDeclarations_x86.hpp index 3e0e1d5bf074a..61deb113d6fa2 100644 --- a/src/hotspot/cpu/x86/stubDeclarations_x86.hpp +++ b/src/hotspot/cpu/x86/stubDeclarations_x86.hpp @@ -161,6 +161,12 @@ do_arch_entry(x86, compiler, pshuffle_byte_flip_mask, \ pshuffle_byte_flip_mask_addr, \ pshuffle_byte_flip_mask_addr) \ + do_arch_entry(x86, compiler, pshuffle_byte_flip_mask, \ + pshuffle_byte_flip_mask_00ba_addr, \ + pshuffle_byte_flip_mask_00ba_addr) \ + do_arch_entry(x86, compiler, pshuffle_byte_flip_mask, \ + pshuffle_byte_flip_mask_dc00_addr, \ + pshuffle_byte_flip_mask_dc00_addr) \ /* x86_64 exposes these 3 stubs via a generic entry array */ \ /* other arches use arch-specific entries */ \ /* this really needs rationalising */ \ @@ -168,6 +174,9 @@ do_stub(compiler, string_indexof_linear_uu) \ do_stub(compiler, string_indexof_linear_ul) \ do_stub(compiler, pshuffle_byte_flip_mask_sha512) \ + do_arch_entry(x86, compiler, pshuffle_byte_flip_mask_sha512, \ + pshuffle_byte_flip_mask_ymm_lo_addr_sha512, \ + pshuffle_byte_flip_mask_ymm_lo_addr_sha512) \ do_arch_entry(x86, compiler, pshuffle_byte_flip_mask_sha512, \ pshuffle_byte_flip_mask_addr_sha512, \ pshuffle_byte_flip_mask_addr_sha512) \ diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp index efb0411aa39df..e04729f070722 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp @@ -39,6 +39,9 @@ #include "runtime/sharedRuntime.hpp" #include "runtime/stubRoutines.hpp" #include "stubGenerator_x86_64.hpp" +#ifdef COMPILER1 +#include "c1/c1_LIRAssembler.hpp" +#endif #ifdef COMPILER2 #include "opto/runtime.hpp" #include "opto/c2_globals.hpp" @@ -188,6 +191,19 @@ address StubGenerator::generate_call_stub(address& return_address) { (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off, "adjust this code"); StubId stub_id = StubId::stubgen_call_stub_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 2, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + // this stub has an extra return entry which we need to retrieve + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == 1, "expected 1 extra entry"); + return_address = entries.at(0); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -298,6 +314,7 @@ address StubGenerator::generate_call_stub(address& return_address) { BLOCK_COMMENT("call_stub_return_address:"); return_address = __ pc(); + entries.append(return_address); // store result depending on type (everything that is not // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT) @@ -394,6 +411,9 @@ address StubGenerator::generate_call_stub(address& return_address) { __ movdbl(Address(c_rarg0, 0), xmm0); __ jmp(exit); + // record the stub entry and end plus the auxiliary entry + store_archive_data(stub_id, start, __ pc(), &entries); + return start; } @@ -411,6 +431,15 @@ address StubGenerator::generate_call_stub(address& return_address) { address StubGenerator::generate_catch_exception() { StubId stub_id = StubId::stubgen_catch_exception_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -442,7 +471,9 @@ address StubGenerator::generate_catch_exception() { __ verify_oop(rax); __ movptr(Address(r15_thread, Thread::pending_exception_offset()), rax); - __ lea(rscratch1, ExternalAddress((address)__FILE__)); + // special case -- add file name string to AOT address table + address file = (address)AOTCodeCache::add_C_string(__FILE__); + __ lea(rscratch1, ExternalAddress(file)); __ movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1); __ movl(Address(r15_thread, Thread::exception_line_offset()), (int) __LINE__); @@ -451,6 +482,9 @@ address StubGenerator::generate_catch_exception() { "_call_stub_return_address must have been generated before"); __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address)); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -467,6 +501,14 @@ address StubGenerator::generate_catch_exception() { address StubGenerator::generate_forward_exception() { StubId stub_id = StubId::stubgen_forward_exception_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -521,6 +563,9 @@ address StubGenerator::generate_forward_exception() { __ verify_oop(rax); __ jmp(rbx); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -531,12 +576,23 @@ address StubGenerator::generate_forward_exception() { // Result: address StubGenerator::generate_orderaccess_fence() { StubId stub_id = StubId::stubgen_fence_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); __ membar(Assembler::StoreLoad); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -550,6 +606,14 @@ address StubGenerator::generate_orderaccess_fence() { address StubGenerator::generate_verify_mxcsr() { StubId stub_id = StubId::stubgen_verify_mxcsr_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -574,11 +638,22 @@ address StubGenerator::generate_verify_mxcsr() { __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_f2i_fixup() { StubId stub_id = StubId::stubgen_f2i_fixup_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); Address inout(rsp, 5 * wordSize); // return address + 4 saves @@ -613,11 +688,22 @@ address StubGenerator::generate_f2i_fixup() { __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_f2l_fixup() { StubId stub_id = StubId::stubgen_f2l_fixup_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); Address inout(rsp, 5 * wordSize); // return address + 4 saves address start = __ pc(); @@ -651,11 +737,22 @@ address StubGenerator::generate_f2l_fixup() { __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_d2i_fixup() { StubId stub_id = StubId::stubgen_d2i_fixup_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); Address inout(rsp, 6 * wordSize); // return address + 5 saves @@ -699,11 +796,22 @@ address StubGenerator::generate_d2i_fixup() { __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_d2l_fixup() { StubId stub_id = StubId::stubgen_d2l_fixup_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); Address inout(rsp, 6 * wordSize); // return address + 5 saves @@ -747,12 +855,23 @@ address StubGenerator::generate_d2l_fixup() { __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_count_leading_zeros_lut() { - __ align64(); StubId stub_id = StubId::stubgen_vector_count_leading_zeros_lut_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -765,12 +884,23 @@ address StubGenerator::generate_count_leading_zeros_lut() { __ emit_data64(0x0101010102020304, relocInfo::none); __ emit_data64(0x0000000000000000, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_popcount_avx_lut() { - __ align64(); StubId stub_id = StubId::stubgen_vector_popcount_lut_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -783,12 +913,23 @@ address StubGenerator::generate_popcount_avx_lut() { __ emit_data64(0x0302020102010100, relocInfo::none); __ emit_data64(0x0403030203020201, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_iota_indices() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_vector_iota_indices_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); // B @@ -845,12 +986,24 @@ address StubGenerator::generate_iota_indices() { __ emit_data64(0x4014000000000000, relocInfo::none); // 5.0d __ emit_data64(0x4018000000000000, relocInfo::none); // 6.0d __ emit_data64(0x401c000000000000, relocInfo::none); // 7.0d + + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_vector_reverse_bit_lut() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_vector_reverse_bit_lut_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -863,12 +1016,23 @@ address StubGenerator::generate_vector_reverse_bit_lut() { __ emit_data64(0x0E060A020C040800, relocInfo::none); __ emit_data64(0x0F070B030D050901, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_vector_reverse_byte_perm_mask_long() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_long_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -881,12 +1045,23 @@ address StubGenerator::generate_vector_reverse_byte_perm_mask_long() { __ emit_data64(0x0001020304050607, relocInfo::none); __ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_vector_reverse_byte_perm_mask_int() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_int_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -899,12 +1074,23 @@ address StubGenerator::generate_vector_reverse_byte_perm_mask_int() { __ emit_data64(0x0405060700010203, relocInfo::none); __ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_vector_reverse_byte_perm_mask_short() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_short_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -917,12 +1103,23 @@ address StubGenerator::generate_vector_reverse_byte_perm_mask_short() { __ emit_data64(0x0607040502030001, relocInfo::none); __ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_vector_byte_shuffle_mask() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_vector_byte_shuffle_mask_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -931,10 +1128,21 @@ address StubGenerator::generate_vector_byte_shuffle_mask() { __ emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none); __ emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_fp_mask(StubId stub_id, int64_t mask) { + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -942,6 +1150,9 @@ address StubGenerator::generate_fp_mask(StubId stub_id, int64_t mask) { __ emit_data64( mask, relocInfo::none ); __ emit_data64( mask, relocInfo::none ); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -957,6 +1168,14 @@ address StubGenerator::generate_compress_perm_table(StubId stub_id) { default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -997,6 +1216,9 @@ address StubGenerator::generate_compress_perm_table(StubId stub_id) { } } } + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1012,6 +1234,14 @@ address StubGenerator::generate_expand_perm_table(StubId stub_id) { default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1050,10 +1280,21 @@ address StubGenerator::generate_expand_perm_table(StubId stub_id) { } } } + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_vector_mask(StubId stub_id, int64_t mask) { + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1067,12 +1308,23 @@ address StubGenerator::generate_vector_mask(StubId stub_id, int64_t mask) { __ emit_data64(mask, relocInfo::none); __ emit_data64(mask, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_vector_byte_perm_mask() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_vector_byte_perm_mask_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1085,10 +1337,21 @@ address StubGenerator::generate_vector_byte_perm_mask() { __ emit_data64(0x0000000000000004, relocInfo::none); __ emit_data64(0x0000000000000006, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_vector_fp_mask(StubId stub_id, int64_t mask) { + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1102,6 +1365,9 @@ address StubGenerator::generate_vector_fp_mask(StubId stub_id, int64_t mask) { __ emit_data64(mask, relocInfo::none); __ emit_data64(mask, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1110,6 +1376,14 @@ address StubGenerator::generate_vector_custom_i32(StubId stub_id, Assembler::Avx int32_t val4, int32_t val5, int32_t val6, int32_t val7, int32_t val8, int32_t val9, int32_t val10, int32_t val11, int32_t val12, int32_t val13, int32_t val14, int32_t val15) { + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1135,6 +1409,9 @@ address StubGenerator::generate_vector_custom_i32(StubId stub_id, Assembler::Avx __ emit_data(val15, relocInfo::none, 0); } } + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1156,6 +1433,14 @@ address StubGenerator::generate_vector_custom_i32(StubId stub_id, Assembler::Avx // * = popped on exit address StubGenerator::generate_verify_oop() { StubId stub_id = StubId::stubgen_verify_oop_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1235,6 +1520,9 @@ address StubGenerator::generate_verify_oop() { __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64))); __ hlt(); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1350,10 +1638,16 @@ void StubGenerator::restore_argument_regs(BasicType type) { address StubGenerator::generate_data_cache_writeback() { const Register src = c_rarg0; // source address - - __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_data_cache_writeback_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1363,15 +1657,24 @@ address StubGenerator::generate_data_cache_writeback() { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_data_cache_writeback_sync() { const Register is_pre = c_rarg0; // pre or post sync - - __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); // pre wbsync is a no-op @@ -1388,6 +1691,9 @@ address StubGenerator::generate_data_cache_writeback_sync() { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1405,6 +1711,14 @@ address StubGenerator::generate_md5_implCompress(StubId stub_id) { default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1437,30 +1751,55 @@ address StubGenerator::generate_md5_implCompress(StubId stub_id) { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_upper_word_mask() { - __ align64(); StubId stub_id = StubId::stubgen_upper_word_mask_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); __ emit_data64(0x0000000000000000, relocInfo::none); __ emit_data64(0xFFFFFFFF00000000, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_shuffle_byte_flip_mask() { - __ align64(); StubId stub_id = StubId::stubgen_shuffle_byte_flip_mask_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); __ emit_data64(0x0001020304050607, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1478,6 +1817,14 @@ address StubGenerator::generate_sha1_implCompress(StubId stub_id) { default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1509,15 +1856,34 @@ address StubGenerator::generate_sha1_implCompress(StubId stub_id) { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } -address StubGenerator::generate_pshuffle_byte_flip_mask() { - __ align64(); +address StubGenerator::generate_pshuffle_byte_flip_mask(address& entry_00ba, address& entry_dc00) { StubId stub_id = StubId::stubgen_pshuffle_byte_flip_mask_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 3, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == entry_count - 1, + "unexpected extra entry count %d", entries.length()); + entry_00ba = entries.at(0); + entry_dc00 = entries.at(1); + assert(VM_Version::supports_avx2() == (entry_00ba != nullptr && entry_dc00 != nullptr), + "entries cannot be null when avx2 is enabled"); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); - + address entry2 = nullptr; + address entry3 = nullptr; __ emit_data64(0x0405060700010203, relocInfo::none); __ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none); @@ -1525,37 +1891,68 @@ address StubGenerator::generate_pshuffle_byte_flip_mask() { __ emit_data64(0x0405060700010203, relocInfo::none); // second copy __ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none); // _SHUF_00BA + entry2 = __ pc(); __ emit_data64(0x0b0a090803020100, relocInfo::none); __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); __ emit_data64(0x0b0a090803020100, relocInfo::none); __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); // _SHUF_DC00 + entry3 = __ pc(); __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); __ emit_data64(0x0b0a090803020100, relocInfo::none); __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); __ emit_data64(0x0b0a090803020100, relocInfo::none); } + // have to track the 2nd and 3rd entries even if they are null + entry_00ba = entry2; + entries.push(entry_00ba); + entry_dc00 = entry3; + entries.push(entry_dc00); + + // record the stub entry and end plus all the auxiliary entries + store_archive_data(stub_id, start, __ pc(), &entries); return start; } //Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb. -address StubGenerator::generate_pshuffle_byte_flip_mask_sha512() { - __ align32(); +address StubGenerator::generate_pshuffle_byte_flip_mask_sha512(address& entry_ymm_lo) { StubId stub_id = StubId::stubgen_pshuffle_byte_flip_mask_sha512_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 2, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == entry_count - 1, + "unexpected extra entry count %d", entries.length()); + entry_ymm_lo = entries.at(0); + assert(VM_Version::supports_avx2() == (entry_ymm_lo != nullptr), + "entry cannot be null when avx2 is enabled"); + return start; + } + __ align32(); StubCodeMark mark(this, stub_id); address start = __ pc(); - + address entry2 = nullptr; if (VM_Version::supports_avx2()) { __ emit_data64(0x0001020304050607, relocInfo::none); // PSHUFFLE_BYTE_FLIP_MASK __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); __ emit_data64(0x1011121314151617, relocInfo::none); __ emit_data64(0x18191a1b1c1d1e1f, relocInfo::none); + // capture 2nd entry + entry2 = __ pc(); __ emit_data64(0x0000000000000000, relocInfo::none); //MASK_YMM_LO __ emit_data64(0x0000000000000000, relocInfo::none); __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); } + // have to track the 2nd entry even if it is null + entry_ymm_lo = entry2; + entries.push(entry2); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc(), &entries); return start; } @@ -1575,6 +1972,14 @@ address StubGenerator::generate_sha256_implCompress(StubId stub_id) { ShouldNotReachHere(); } assert(VM_Version::supports_sha() || VM_Version::supports_avx2(), ""); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1612,6 +2017,9 @@ address StubGenerator::generate_sha256_implCompress(StubId stub_id) { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1629,6 +2037,14 @@ address StubGenerator::generate_sha512_implCompress(StubId stub_id) { } assert(VM_Version::supports_avx2(), ""); assert(VM_Version::supports_bmi2() || VM_Version::supports_sha512(), ""); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1660,12 +2076,23 @@ address StubGenerator::generate_sha512_implCompress(StubId stub_id) { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_shuffle_addr() { - __ align64(); StubId stub_id = StubId::stubgen_shuffle_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1680,12 +2107,23 @@ address StubGenerator::base64_shuffle_addr() { __ emit_data64(0x2829272825262425, relocInfo::none); __ emit_data64(0x2e2f2d2e2b2c2a2b, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_avx2_shuffle_addr() { - __ align32(); StubId stub_id = StubId::stubgen_avx2_shuffle_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align32(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1694,12 +2132,23 @@ address StubGenerator::base64_avx2_shuffle_addr() { __ emit_data64(0x0405030401020001, relocInfo::none); __ emit_data64(0x0a0b090a07080607, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_avx2_input_mask_addr() { - __ align32(); StubId stub_id = StubId::stubgen_avx2_input_mask_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align32(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1708,12 +2157,23 @@ address StubGenerator::base64_avx2_input_mask_addr() { __ emit_data64(0x8000000080000000, relocInfo::none); __ emit_data64(0x8000000080000000, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_avx2_lut_addr() { - __ align32(); StubId stub_id = StubId::stubgen_avx2_lut_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align32(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1728,12 +2188,23 @@ address StubGenerator::base64_avx2_lut_addr() { __ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); __ emit_data64(0x000020effcfcfcfc, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_encoding_table_addr() { - __ align64(); StubId stub_id = StubId::stubgen_encoding_table_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1757,6 +2228,9 @@ address StubGenerator::base64_encoding_table_addr() { __ emit_data64(0x333231307a797877, relocInfo::none); __ emit_data64(0x5f2d393837363534, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1766,8 +2240,16 @@ address StubGenerator::base64_encoding_table_addr() { // boolean isURL) { address StubGenerator::generate_base64_encodeBlock() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_base64_encodeBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2144,13 +2626,24 @@ address StubGenerator::generate_base64_encodeBlock() __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } // base64 AVX512vbmi tables address StubGenerator::base64_vbmi_lookup_lo_addr() { - __ align64(); StubId stub_id = StubId::stubgen_lookup_lo_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2165,12 +2658,23 @@ address StubGenerator::base64_vbmi_lookup_lo_addr() { __ emit_data64(0x3b3a393837363534, relocInfo::none); __ emit_data64(0x8080808080803d3c, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_vbmi_lookup_hi_addr() { - __ align64(); StubId stub_id = StubId::stubgen_lookup_hi_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2185,11 +2689,22 @@ address StubGenerator::base64_vbmi_lookup_hi_addr() { __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); __ emit_data64(0x8080808080333231, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_vbmi_lookup_lo_url_addr() { - __ align64(); StubId stub_id = StubId::stubgen_lookup_lo_base64url_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2204,12 +2719,23 @@ address StubGenerator::base64_vbmi_lookup_lo_url_addr() { __ emit_data64(0x3b3a393837363534, relocInfo::none); __ emit_data64(0x8080808080803d3c, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_vbmi_lookup_hi_url_addr() { - __ align64(); StubId stub_id = StubId::stubgen_lookup_hi_base64url_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2224,12 +2750,23 @@ address StubGenerator::base64_vbmi_lookup_hi_url_addr() { __ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); __ emit_data64(0x8080808080333231, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_vbmi_pack_vec_addr() { - __ align64(); StubId stub_id = StubId::stubgen_pack_vec_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2244,12 +2781,23 @@ address StubGenerator::base64_vbmi_pack_vec_addr() { __ emit_data64(0x0000000000000000, relocInfo::none); __ emit_data64(0x0000000000000000, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_vbmi_join_0_1_addr() { - __ align64(); StubId stub_id = StubId::stubgen_join_0_1_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2264,12 +2812,23 @@ address StubGenerator::base64_vbmi_join_0_1_addr() { __ emit_data64(0x494a444546404142, relocInfo::none); __ emit_data64(0x565051524c4d4e48, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_vbmi_join_1_2_addr() { - __ align64(); StubId stub_id = StubId::stubgen_join_1_2_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2284,12 +2843,23 @@ address StubGenerator::base64_vbmi_join_1_2_addr() { __ emit_data64(0x5c5d5e58595a5455, relocInfo::none); __ emit_data64(0x696a646566606162, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_vbmi_join_2_3_addr() { - __ align64(); StubId stub_id = StubId::stubgen_join_2_3_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2304,12 +2874,23 @@ address StubGenerator::base64_vbmi_join_2_3_addr() { __ emit_data64(0x767071726c6d6e68, relocInfo::none); __ emit_data64(0x7c7d7e78797a7475, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_AVX2_decode_tables_addr() { - __ align64(); StubId stub_id = StubId::stubgen_avx2_decode_tables_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2339,12 +2920,23 @@ address StubGenerator::base64_AVX2_decode_tables_addr() { // merge multiplier __ emit_data(0x00011000, relocInfo::none, 0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_AVX2_decode_LUT_tables_addr() { - __ align64(); StubId stub_id = StubId::stubgen_avx2_decode_lut_tables_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align64(); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2380,11 +2972,22 @@ address StubGenerator::base64_AVX2_decode_LUT_tables_addr() { __ emit_data64(0x0804080402011010, relocInfo::none); __ emit_data64(0x1010101010101010, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::base64_decoding_table_addr() { StubId stub_id = StubId::stubgen_decoding_table_base64_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2455,6 +3058,9 @@ address StubGenerator::base64_decoding_table_addr() { __ emit_data64(0xffffffffffffffff, relocInfo::none); __ emit_data64(0xffffffffffffffff, relocInfo::none); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2466,8 +3072,16 @@ address StubGenerator::base64_decoding_table_addr() { // Intrinsic function prototype in Base64.java: // private void decodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL, isMIME) { address StubGenerator::generate_base64_decodeBlock() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_base64_decodeBlock_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2982,6 +3596,9 @@ address StubGenerator::generate_base64_decodeBlock() { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3000,8 +3617,16 @@ address StubGenerator::generate_base64_decodeBlock() { address StubGenerator::generate_updateBytesCRC32() { assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_updateBytesCRC32_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3039,6 +3664,9 @@ address StubGenerator::generate_updateBytesCRC32() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3057,8 +3685,16 @@ address StubGenerator::generate_updateBytesCRC32() { */ address StubGenerator::generate_updateBytesCRC32C(bool is_pclmulqdq_supported) { assert(UseCRC32CIntrinsics, "need SSE4_2"); - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_updateBytesCRC32C_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3120,6 +3756,9 @@ address StubGenerator::generate_updateBytesCRC32C(bool is_pclmulqdq_supported) { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3138,8 +3777,16 @@ address StubGenerator::generate_updateBytesCRC32C(bool is_pclmulqdq_supported) { * rsp+40 - z address */ address StubGenerator::generate_multiplyToLen() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_multiplyToLen_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3179,6 +3826,9 @@ address StubGenerator::generate_multiplyToLen() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3195,8 +3845,16 @@ address StubGenerator::generate_multiplyToLen() { * rax - int >= mismatched index, < 0 bitwise complement of tail */ address StubGenerator::generate_vectorizedMismatch() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_vectorizedMismatch_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3232,6 +3890,9 @@ address StubGenerator::generate_vectorizedMismatch() { __ leave(); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3247,8 +3908,16 @@ address StubGenerator::generate_vectorizedMismatch() { */ address StubGenerator::generate_squareToLen() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_squareToLen_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3279,12 +3948,23 @@ address StubGenerator::generate_squareToLen() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_method_entry_barrier() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_method_entry_barrier_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3356,6 +4036,9 @@ address StubGenerator::generate_method_entry_barrier() { __ movptr(rsp, Address(rsp, 0)); // new rsp was written in the barrier __ jmp(Address(rsp, -1 * wordSize)); // jmp target should be callers verified_entry_point + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3373,8 +4056,16 @@ address StubGenerator::generate_method_entry_barrier() { * rsp+40 - k */ address StubGenerator::generate_mulAdd() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_mulAdd_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3411,12 +4102,23 @@ address StubGenerator::generate_mulAdd() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } address StubGenerator::generate_bigIntegerRightShift() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_bigIntegerRightShiftWorker_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3534,6 +4236,9 @@ address StubGenerator::generate_bigIntegerRightShift() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3551,8 +4256,16 @@ address StubGenerator::generate_bigIntegerRightShift() { * rsp40 - numIter */ address StubGenerator::generate_bigIntegerLeftShift() { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_bigIntegerLeftShiftWorker_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3659,6 +4372,9 @@ address StubGenerator::generate_bigIntegerLeftShift() { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3708,6 +4424,14 @@ void StubGenerator::generate_libm_stubs() { */ address StubGenerator::generate_float16ToFloat() { StubId stub_id = StubId::stubgen_hf2f_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3720,6 +4444,9 @@ address StubGenerator::generate_float16ToFloat() { __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3734,6 +4461,14 @@ address StubGenerator::generate_float16ToFloat() { */ address StubGenerator::generate_floatToFloat16() { StubId stub_id = StubId::stubgen_f2hf_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3746,6 +4481,9 @@ address StubGenerator::generate_floatToFloat16() { __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3775,6 +4513,14 @@ address StubGenerator::generate_cont_thaw(StubId stub_id) { default: ShouldNotReachHere(); } + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3889,6 +4635,9 @@ address StubGenerator::generate_cont_thaw(StubId stub_id) { __ ret(0); } + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3909,6 +4658,14 @@ address StubGenerator::generate_cont_returnBarrier_exception() { address StubGenerator::generate_cont_preempt_stub() { if (!Continuations::enabled()) return nullptr; StubId stub_id = StubId::stubgen_cont_preempt_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3934,12 +4691,23 @@ address StubGenerator::generate_cont_preempt_stub() { __ movptr(rscratch1, ExternalAddress(ContinuationEntry::thaw_call_pc_address())); __ jmp(rscratch1); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } // exception handler for upcall stubs address StubGenerator::generate_upcall_stub_exception_handler() { StubId stub_id = StubId::stubgen_upcall_stub_exception_handler_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3953,6 +4721,9 @@ address StubGenerator::generate_upcall_stub_exception_handler() { __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, UpcallLinker::handle_uncaught_exception))); __ should_not_reach_here(); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -3961,6 +4732,14 @@ address StubGenerator::generate_upcall_stub_exception_handler() { // rbx = result address StubGenerator::generate_upcall_stub_load_target() { StubId stub_id = StubId::stubgen_upcall_stub_load_target_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -3981,6 +4760,21 @@ address StubGenerator::generate_upcall_stub_load_target() { void StubGenerator::generate_lookup_secondary_supers_table_stub() { StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_id; + GrowableArray
entries; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == Klass::SECONDARY_SUPERS_TABLE_SIZE, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, &entries); + assert(entries.length() == Klass::SECONDARY_SUPERS_TABLE_SIZE - 1, + "unexpected extra entry count %d", entries.length()); + StubRoutines::_lookup_secondary_supers_table_stubs[0] = start; + for (int slot = 1; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) { + StubRoutines::_lookup_secondary_supers_table_stubs[0] = entries.at(slot - 1); + } + return; + } StubCodeMark mark(this, stub_id); const Register @@ -3988,19 +4782,38 @@ void StubGenerator::generate_lookup_secondary_supers_table_stub() { r_sub_klass = rsi, result = rdi; + address start; for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) { - StubRoutines::_lookup_secondary_supers_table_stubs[slot] = __ pc(); + address next_entry = __ pc(); + if (slot == 0) { + start = next_entry; + } else { + entries.append(next_entry); + } + StubRoutines::_lookup_secondary_supers_table_stubs[slot] = next_entry; __ lookup_secondary_supers_table_const(r_sub_klass, r_super_klass, rdx, rcx, rbx, r11, // temps result, slot); __ ret(0); } + + // record the stub entry and end plus all the auxiliary entries + store_archive_data(stub_id, start, __ pc(), &entries); } // Slow path implementation for UseSecondarySupersTable. address StubGenerator::generate_lookup_secondary_supers_table_slow_path_stub() { StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_slow_path_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -4025,6 +4838,9 @@ address StubGenerator::generate_lookup_secondary_supers_table_slow_path_stub() { __ movl(result, 0); __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -4232,6 +5048,8 @@ void StubGenerator::generate_compiler_stubs() { } if (UseSHA256Intrinsics) { + address entry2 = nullptr; + address entry3 = nullptr; StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256; char* dst = (char*)StubRoutines::x86::_k256_W; char* src = (char*)StubRoutines::x86::_k256; @@ -4240,14 +5058,18 @@ void StubGenerator::generate_compiler_stubs() { memcpy(dst + 32 * ii + 16, src + 16 * ii, 16); } StubRoutines::x86::_k256_W_adr = (address)StubRoutines::x86::_k256_W; - StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask(); + StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask(entry2, entry3); + StubRoutines::x86::_pshuffle_byte_flip_mask_00ba_addr = entry2; + StubRoutines::x86::_pshuffle_byte_flip_mask_dc00_addr = entry3; StubRoutines::_sha256_implCompress = generate_sha256_implCompress(StubId::stubgen_sha256_implCompress_id); StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(StubId::stubgen_sha256_implCompressMB_id); } if (UseSHA512Intrinsics) { + address entry2 = nullptr; StubRoutines::x86::_k512_W_addr = (address)StubRoutines::x86::_k512_W; - StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512(); + StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512(entry2); + StubRoutines::x86::_pshuffle_byte_flip_mask_ymm_lo_addr_sha512 = entry2; StubRoutines::_sha512_implCompress = generate_sha512_implCompress(StubId::stubgen_sha512_implCompress_id); StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(StubId::stubgen_sha512_implCompressMB_id); } @@ -4325,7 +5147,7 @@ void StubGenerator::generate_compiler_stubs() { #endif // COMPILER2_OR_JVMCI } -StubGenerator::StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) { +StubGenerator::StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) { switch(blob_id) { case BlobId::stubgen_preuniverse_id: generate_preuniverse_stubs(); @@ -4348,8 +5170,47 @@ StubGenerator::StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerat }; } -void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) { - StubGenerator g(code, blob_id); +// publish addresses of static data defined in this file and in other +// stubgen stub generator files +void StubGenerator::init_AOTAddressTable(GrowableArray
& external_addresses) { + init_AOTAddressTable_adler(external_addresses); + init_AOTAddressTable_aes(external_addresses); + init_AOTAddressTable_cbrt(external_addresses); + init_AOTAddressTable_chacha(external_addresses); + // constants publishes for all of address use by cos and almost all of sin + init_AOTAddressTable_constants(external_addresses); + init_AOTAddressTable_dilithium(external_addresses); + init_AOTAddressTable_exp(external_addresses); + init_AOTAddressTable_fmod(external_addresses); + init_AOTAddressTable_ghash(external_addresses); + init_AOTAddressTable_kyber(external_addresses); + init_AOTAddressTable_log(external_addresses); + init_AOTAddressTable_poly1305(external_addresses); + init_AOTAddressTable_poly_mont(external_addresses); + init_AOTAddressTable_pow(external_addresses); + init_AOTAddressTable_sha3(external_addresses); + init_AOTAddressTable_sin(external_addresses); + init_AOTAddressTable_sinh(external_addresses); + init_AOTAddressTable_tan(external_addresses); + init_AOTAddressTable_tanh(external_addresses); +} + +void StubGenerator_AOTAddressTable_init() { + ResourceMark rm; + GrowableArray
external_addresses; + // publish static addresses referred to by main x86 generator and + // auxiliary x86 generators + StubGenerator::init_AOTAddressTable(external_addresses); + // publish external data addresses defined in nested x86 class + StubRoutines::x86::init_AOTAddressTable(external_addresses); +#ifdef COMPILER1 + LIR_Assembler::init_AOTAddressTable(external_addresses); +#endif + AOTCodeCache::publish_external_addresses(external_addresses); +} + +void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) { + StubGenerator g(code, blob_id, stub_data); } #undef __ diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp index 36315535d1607..b72a965ba905a 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp @@ -303,11 +303,11 @@ class StubGenerator: public StubCodeGenerator { address generate_sha512_implCompress(StubId stub_id); // Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb. - address generate_pshuffle_byte_flip_mask_sha512(); + address generate_pshuffle_byte_flip_mask_sha512(address& entry_ymm_lo); address generate_upper_word_mask(); address generate_shuffle_byte_flip_mask(); - address generate_pshuffle_byte_flip_mask(); + address generate_pshuffle_byte_flip_mask(address& entry_00ba, address& entry_dc0); // AES intrinsic stubs @@ -637,8 +637,30 @@ class StubGenerator: public StubCodeGenerator { void generate_compiler_stubs(); void generate_final_stubs(); + static void init_AOTAddressTable_adler(GrowableArray
& external_addresses); + static void init_AOTAddressTable_aes(GrowableArray
& external_addresses); + static void init_AOTAddressTable_cbrt(GrowableArray
& external_addresses); + static void init_AOTAddressTable_(GrowableArray
& external_addresses); + static void init_AOTAddressTable_chacha(GrowableArray
& external_addresses); + static void init_AOTAddressTable_constants(GrowableArray
& external_addresses); + static void init_AOTAddressTable_dilithium(GrowableArray
& external_addresses); + static void init_AOTAddressTable_exp(GrowableArray
& external_addresses); + static void init_AOTAddressTable_fmod(GrowableArray
& external_addresses); + static void init_AOTAddressTable_ghash(GrowableArray
& external_addresses); + static void init_AOTAddressTable_kyber(GrowableArray
& external_addresses); + static void init_AOTAddressTable_log(GrowableArray
& external_addresses); + static void init_AOTAddressTable_poly1305(GrowableArray
& external_addresses); + static void init_AOTAddressTable_poly_mont(GrowableArray
& external_addresses); + static void init_AOTAddressTable_pow(GrowableArray
& external_addresses); + static void init_AOTAddressTable_sha3(GrowableArray
& external_addresses); + static void init_AOTAddressTable_sin(GrowableArray
& external_addresses); + static void init_AOTAddressTable_sinh(GrowableArray
& external_addresses); + static void init_AOTAddressTable_tan(GrowableArray
& external_addresses); + static void init_AOTAddressTable_tanh(GrowableArray
& external_addresses); + public: - StubGenerator(CodeBuffer* code, BlobId blob_id); + StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data); + static void init_AOTAddressTable(GrowableArray
& external_addresses); }; #endif // CPU_X86_STUBGENERATOR_X86_64_HPP diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp index 2799997a761da..c5aa877b4fcf7 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp @@ -338,3 +338,11 @@ address StubGenerator::generate_updateBytesAdler32() { } #undef __ + +void StubGenerator::init_AOTAddressTable_adler(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)(addr)) + ADD(ADLER32_ASCALE_TABLE); + ADD(ADLER32_SHUF0_TABLE); + ADD(ADLER32_SHUF1_TABLE); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp index f0726ded7e546..bb991e30b76af 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp @@ -4093,3 +4093,25 @@ void StubGenerator::aesgcm_avx2(Register in, Register len, Register ct, Register } #undef __ + +void StubGenerator::init_AOTAddressTable_aes(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)(addr)) + ADD(key_shuffle_mask_addr()); + ADD(counter_shuffle_mask_addr()); + ADD(counter_mask_linc0_addr()); + ADD(counter_mask_linc1_addr()); + ADD(counter_mask_linc1f_addr()); + ADD(counter_mask_linc2_addr()); + ADD(counter_mask_linc2f_addr()); + ADD(counter_mask_linc4_addr()); + ADD(counter_mask_linc8_addr()); + ADD(counter_mask_linc16_addr()); + ADD(counter_mask_linc32_addr()); + ADD(counter_mask_ones_addr()); + ADD(ghash_polynomial_reduction_addr()); + ADD(ghash_polynomial_two_one_addr()); + ADD(counter_mask_addbe_4444_addr()); + ADD(counter_mask_addbe_1234_addr()); + ADD(counter_mask_add_1234_addr()); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp index d53fafafdb440..c9649fbfa3933 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp @@ -570,6 +570,32 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres default: ShouldNotReachHere(); } + GrowableArray
entries; + GrowableArray
extras; + bool add_extras = !is_oop && !aligned; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = 3 * (add_extras ? 2 : 0); // 0/2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + if (add_extras) { + // restore 1 x UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 1, end); + } + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -596,6 +622,7 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -794,6 +821,18 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres arraycopy_avx3_large(to, from, temp1, temp2, temp3, temp4, count, xmm1, xmm2, xmm3, xmm4, shift); __ jmp(L_finish); } + // retrieve the registered handler addresses + address end = __ pc(); + if (add_extras) { + retrieve_unsafe_access_handlers(start, end, extras); + } + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + return start; } @@ -908,7 +947,32 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres default: ShouldNotReachHere(); } - + GrowableArray
entries; + GrowableArray
extras; + bool add_extras = !is_oop && !aligned; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = 3 * (add_extras ? 1 : 0); // 0/1 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + if (add_extras) { + // restore 1 x UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 1, end); + } + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -933,6 +997,7 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -959,7 +1024,7 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres int threshold[] = { 4096, 2048, 1024, 512}; // UnsafeMemoryAccess page error: continue after unsafe access - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); // 'from', 'to' and 'count' are now valid // temp1 holds remaining count. @@ -1073,6 +1138,18 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // retrieve the registered handler addresses + address end = __ pc(); + if (add_extras) { + retrieve_unsafe_access_handlers(start, end, extras); + } + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + return start; } @@ -1387,6 +1464,28 @@ address StubGenerator::generate_disjoint_byte_copy(address* entry) { return generate_disjoint_copy_avx3_masked(stub_id, entry); } #endif + GrowableArray
entries; + GrowableArray
extras; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = (2 * 3); // 2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, &extras); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1409,6 +1508,7 @@ address StubGenerator::generate_disjoint_byte_copy(address* entry) { if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -1478,6 +1578,17 @@ __ BIND(L_exit); copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, T_BYTE); __ jmp(L_copy_4_bytes); } + + // retrieve the registered handler addresses + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, &extras); + return start; } @@ -1505,6 +1616,28 @@ address StubGenerator::generate_conjoint_byte_copy(address nooverlap_target, add return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target); } #endif + GrowableArray
entries; + GrowableArray
extras; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = (2 * 3); // 2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, &extras); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1522,6 +1655,7 @@ address StubGenerator::generate_conjoint_byte_copy(address nooverlap_target, add if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -1588,6 +1722,16 @@ address StubGenerator::generate_conjoint_byte_copy(address nooverlap_target, add __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // retrieve the registered handler addresses + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, &extras); + return start; } @@ -1618,7 +1762,28 @@ address StubGenerator::generate_disjoint_short_copy(address *entry) { return generate_disjoint_copy_avx3_masked(stub_id, entry); } #endif - + GrowableArray
entries; + GrowableArray
extras; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = (2 * 3); // 2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, &extras); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1640,6 +1805,7 @@ address StubGenerator::generate_disjoint_short_copy(address *entry) { if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -1703,6 +1869,16 @@ __ BIND(L_exit); __ jmp(L_copy_4_bytes); } + // retrieve the registered handler addresses + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, &extras); + return start; } @@ -1710,7 +1886,14 @@ __ BIND(L_exit); address StubGenerator::generate_fill(StubId stub_id) { BasicType t; bool aligned; - + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } switch (stub_id) { case StubId::stubgen_jbyte_fill_id: t = T_BYTE; @@ -1763,6 +1946,9 @@ address StubGenerator::generate_fill(StubId stub_id) { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -1790,7 +1976,28 @@ address StubGenerator::generate_conjoint_short_copy(address nooverlap_target, ad return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target); } #endif - + GrowableArray
entries; + GrowableArray
extras; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = (2 * 3); // 2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, &extras); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -1808,6 +2015,7 @@ address StubGenerator::generate_conjoint_short_copy(address nooverlap_target, ad if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -1866,6 +2074,16 @@ address StubGenerator::generate_conjoint_short_copy(address nooverlap_target, ad __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // retrieve the registered handler addresses + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, &extras); + return start; } @@ -1918,6 +2136,32 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e return generate_disjoint_copy_avx3_masked(stub_id, entry); } #endif + GrowableArray
entries; + GrowableArray
extras; + bool add_extras = !is_oop && !aligned; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = 3 * (add_extras ? 2 : 0); // 0/2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + if (add_extras) { + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + } + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -1939,6 +2183,7 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -1959,7 +2204,7 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e { // UnsafeMemoryAccess page error: continue after unsafe access - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); // 'from', 'to' and 'count' are now valid __ movptr(dword_count, count); __ shrptr(count, 1); // count => qword_count @@ -1971,20 +2216,20 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e __ jmp(L_copy_bytes); // Copy trailing qwords - __ BIND(L_copy_8_bytes); + __ BIND(L_copy_8_bytes); __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); __ increment(qword_count); __ jcc(Assembler::notZero, L_copy_8_bytes); // Check for and copy trailing dword - __ BIND(L_copy_4_bytes); + __ BIND(L_copy_4_bytes); __ testl(dword_count, 1); // Only byte test since the value is 0 or 1 __ jccb(Assembler::zero, L_exit); __ movl(rax, Address(end_from, 8)); __ movl(Address(end_to, 8), rax); } -__ BIND(L_exit); + __ BIND(L_exit); address ucme_exit_pc = __ pc(); bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count); restore_arg_regs_using_thread(); @@ -1995,12 +2240,24 @@ __ BIND(L_exit); __ ret(0); { - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, false, ucme_exit_pc); + UnsafeMemoryAccessMark umam(this, add_extras, false, ucme_exit_pc); // Copy in multi-bytes chunks copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_INT); __ jmp(L_copy_4_bytes); } + // retrieve the registered handler addresses + address end = __ pc(); + if (add_extras) { + retrieve_unsafe_access_handlers(start, end, extras); + } + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + return start; } @@ -2049,6 +2306,32 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target); } #endif + bool add_extras = !is_oop && !aligned; + GrowableArray
entries; + GrowableArray
extras; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = 3 * (add_extras ? 2 : 0); // 0/2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + if (add_extras) { + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + } + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -2066,7 +2349,8 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no if (entry != nullptr) { *entry = __ pc(); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + entries.append(*entry); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -2089,7 +2373,7 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no assert_clean_int(count, rax); // Make sure 'count' is clean int. { // UnsafeMemoryAccess page error: continue after unsafe access - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); // 'from', 'to' and 'count' are now valid __ movptr(dword_count, count); __ shrptr(count, 1); // count => qword_count @@ -2104,7 +2388,7 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no __ jmp(L_copy_bytes); // Copy trailing qwords - __ BIND(L_copy_8_bytes); + __ BIND(L_copy_8_bytes); __ movq(rax, Address(from, qword_count, Address::times_8, -8)); __ movq(Address(to, qword_count, Address::times_8, -8), rax); __ decrement(qword_count); @@ -2122,12 +2406,12 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no { // UnsafeMemoryAccess page error: continue after unsafe access - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); // Copy in multi-bytes chunks copy_bytes_backward(from, to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_INT); } -__ BIND(L_exit); + __ BIND(L_exit); bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count); restore_arg_regs_using_thread(); INC_COUNTER_NP(SharedRuntime::_jint_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free @@ -2136,6 +2420,18 @@ __ BIND(L_exit); __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // retrieve the registered handler addresses + address end = __ pc(); + if (add_extras) { + retrieve_unsafe_access_handlers(start, end, extras); + } + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + return start; } @@ -2182,6 +2478,32 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address * return generate_disjoint_copy_avx3_masked(stub_id, entry); } #endif + bool add_extras = !is_oop && !aligned; + GrowableArray
entries; + GrowableArray
extras; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = 3 * (add_extras ? 2 : 0); // 0/2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + if (add_extras) { + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + } + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -2203,6 +2525,7 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address * if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -2223,7 +2546,7 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address * bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count); { // UnsafeMemoryAccess page error: continue after unsafe access - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); // Copy from low to high addresses. Use 'to' as scratch. __ lea(end_from, Address(from, qword_count, Address::times_8, -8)); @@ -2255,7 +2578,7 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address * { // UnsafeMemoryAccess page error: continue after unsafe access - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); // Copy in multi-bytes chunks copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_LONG); } @@ -2271,6 +2594,18 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address * __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // retrieve the registered handler addresses + address end = __ pc(); + if (add_extras) { + retrieve_unsafe_access_handlers(start, end, extras); + } + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + return start; } @@ -2315,6 +2650,32 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target); } #endif + bool add_extras = !is_oop && !aligned; + GrowableArray
entries; + GrowableArray
extras; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int expected_extra_count = 3 * (add_extras ? 2 : 0); // 0/2 x UMAM {start,end,handler} + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + GrowableArray
* extras_ptr = (add_extras ? &extras : nullptr); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + if (add_extras) { + // restore 2 UMAM {start,end,handler} addresses from extras + register_unsafe_access_handlers(extras, 0, 2, end); + } + return start; + } __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); @@ -2331,6 +2692,7 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) BLOCK_COMMENT("Entry:"); } @@ -2352,7 +2714,7 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count); { // UnsafeMemoryAccess page error: continue after unsafe access - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); __ jmp(L_copy_bytes); @@ -2379,7 +2741,7 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n } { // UnsafeMemoryAccess page error: continue after unsafe access - UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true); + UnsafeMemoryAccessMark umam(this, add_extras, true); // Copy in multi-bytes chunks copy_bytes_backward(from, to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_LONG); @@ -2395,6 +2757,19 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + + // retrieve the registered handler addresses + address end = __ pc(); + if (add_extras) { + retrieve_unsafe_access_handlers(start, end, extras); + } + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, entries_ptr, extras_ptr); + return start; } @@ -2450,6 +2825,23 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) { ShouldNotReachHere(); } + GrowableArray
entries; + int expected_entry_count = (entry != nullptr ? 2 : 1); + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == expected_entry_count, "sanity check"); + GrowableArray
* entries_ptr = (entry_count == 1 ? nullptr : &entries); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, entries_ptr); + assert(entries.length() == expected_entry_count - 1, + "unexpected extra entry count %d", entries.length()); + if (entry != nullptr) { + *entry = entries.at(0); + } + return start; + } + Label L_load_element, L_store_element, L_do_card_marks, L_done; // Input registers (after setup_arg_regs) @@ -2504,6 +2896,7 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) { // Caller of this entry point must set up the argument registers. if (entry != nullptr) { *entry = __ pc(); + entries.append(*entry); BLOCK_COMMENT("Entry:"); } @@ -2638,6 +3031,9 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) { __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end plus the no_push entry + store_archive_data(stub_id, start, __ pc(), entries_ptr); + return start; } @@ -2657,6 +3053,19 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) { address StubGenerator::generate_unsafe_copy(address byte_copy_entry, address short_copy_entry, address int_copy_entry, address long_copy_entry) { + StubId stub_id = StubId::stubgen_unsafe_arraycopy_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + // TODO - in debug mode we should save and restore the supplied + // copy entries and assert that the restored entries match the ones + // passed in + load_archive_data(stub_id, start, end); + return start; + } + Label L_long_aligned, L_int_aligned, L_short_aligned; // Input registers (before setup_arg_regs) @@ -2668,7 +3077,6 @@ address StubGenerator::generate_unsafe_copy(address byte_copy_entry, address sho const Register bits = rax; // test copy of low bits __ align(CodeEntryAlignment); - StubId stub_id = StubId::stubgen_unsafe_arraycopy_id; StubCodeMark mark(this, stub_id); address start = __ pc(); @@ -2702,6 +3110,9 @@ address StubGenerator::generate_unsafe_copy(address byte_copy_entry, address sho __ shrptr(size, LogBytesPerLong); // size => qword_count __ jump(RuntimeAddress(long_copy_entry)); + // record the stub entry and end plus + store_archive_data(stub_id, start, __ pc()); + return start; } @@ -2803,8 +3214,23 @@ static void do_setmemory_atomic_loop(USM_TYPE type, Register dest, // to an int, short, or byte fill loop. // address StubGenerator::generate_unsafe_setmemory(address unsafe_byte_fill) { - __ align(CodeEntryAlignment); StubId stub_id = StubId::stubgen_unsafe_setmemory_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + // we expect one set of extra unsafememory access handler entries + GrowableArray
extras; + int expected_extra_count = 3 * 3; + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end, nullptr, &extras); + assert(extras.length() == expected_extra_count, + "unexpected extra entry count %d", extras.length()); + register_unsafe_access_handlers(extras, 0, 3, end); + return start; + } + + __ align(CodeEntryAlignment); StubCodeMark mark(this, stub_id); address start = __ pc(); __ enter(); // required for proper stackwalking of RuntimeStub frame @@ -2896,6 +3322,16 @@ address StubGenerator::generate_unsafe_setmemory(address unsafe_byte_fill) { __ jump(RuntimeAddress(unsafe_byte_fill)); } + // retrieve the registered handler addresses + address end = __ pc(); + retrieve_unsafe_access_handlers(start, end, extras); + assert(extras.length() == expected_extra_count, + "unexpected extra addresses count %d", extras.length()); + + // record the stub entry and end plus the no_push entry and any + // extra handler addresses + store_archive_data(stub_id, start, end, nullptr, &extras); + return start; } @@ -2952,7 +3388,16 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh address int_copy_entry, address oop_copy_entry, address long_copy_entry, address checkcast_copy_entry) { - Label L_failed, L_failed_0, L_objArray; + StubId stub_id = StubId::stubgen_generic_arraycopy_id; + int entry_count = StubInfo::entry_count(stub_id); + assert(entry_count == 1, "sanity check"); + if (find_archive_data(stub_id)) { + address start = nullptr; + address end = nullptr; + load_archive_data(stub_id, start, end); + return start; + } + Label L_failed, L_failed_0, L_skip_failed_0, L_objArray; Label L_copy_shorts, L_copy_ints, L_copy_longs; // Input registers @@ -2968,20 +3413,7 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh const Register rklass_tmp = rdi; // load_klass #endif - { int modulus = CodeEntryAlignment; - int target = modulus - 5; // 5 = sizeof jmp(L_failed) - int advance = target - (__ offset() % modulus); - if (advance < 0) advance += modulus; - if (advance > 0) __ nop(advance); - } - StubId stub_id = StubId::stubgen_generic_arraycopy_id; StubCodeMark mark(this, stub_id); - - // Short-hop target to L_failed. Makes for denser prologue code. - __ BIND(L_failed_0); - __ jmp(L_failed); - assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed"); - __ align(CodeEntryAlignment); address start = __ pc(); @@ -3024,7 +3456,8 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh // if (dst_pos < 0) return -1; __ testl(dst_pos, dst_pos); // dst_pos (32-bits) size_t j4off = __ offset(); - __ jccb(Assembler::negative, L_failed_0); + // skip over the failure trampoline + __ jccb(Assembler::positive, L_skip_failed_0); // The first four tests are very dense code, // but not quite dense enough to put four @@ -3034,6 +3467,13 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh // Make sure of this. guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps"); + // Short-hop target to L_failed. Makes for denser prologue code. + __ BIND(L_failed_0); + __ jmp(L_failed); + + // continue here if first 4 checks pass + __ bind(L_skip_failed_0); + // registers used as temp const Register r11_length = r11; // elements count to copy const Register r10_src_klass = r10; // array klass @@ -3256,6 +3696,9 @@ __ BIND(L_failed); __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + // record the stub entry and end + store_archive_data(stub_id, start, __ pc()); + return start; } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp index 73330dedc0f7c..d386d4a27440f 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp @@ -339,3 +339,24 @@ address StubGenerator::generate_libmCbrt() { } #undef __ +void StubGenerator::init_AOTAddressTable_cbrt(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)(addr)) + ADD(_ABS_MASK); + ADD(_SIG_MASK); + ADD(_EXP_MASK); + ADD(_EXP_MSK2); + ADD(_EXP_MSK3); + ADD(_SCALE63); + ADD(_ZERON); + ADD(_INF); + ADD(_NEG_INF); + address coeff_table = (address)_coeff_table; + ADD(coeff_table); + ADD(coeff_table + 16); + ADD(coeff_table + 32); + ADD(coeff_table + 48); + ADD(_rcp_table); + ADD(_cbrt_table); + ADD(_D_table); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp index 7afaf34e031cc..2366f37112d6f 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp @@ -584,3 +584,10 @@ bVec, } #undef __ +void StubGenerator::init_AOTAddressTable_chacha(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)(addr)) + ADD(CC20_COUNTER_ADD_AVX); + ADD(CC20_COUNTER_ADD_AVX512); + ADD(CC20_LROT_CONSTS); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp index 93fa7e650db6a..af02736347c86 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp @@ -233,3 +233,27 @@ ATTRIBUTE_ALIGNED(16) static const juint _Ctable[] = { }; address StubGenerator::Ctable = (address)_Ctable; +void StubGenerator::init_AOTAddressTable_constants(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)(addr)) + ADD(_ONE); + ADD(_ONEHALF); + ADD(_SIGN_MASK); + ADD(_TWO_POW_55); + ADD(_TWO_POW_M55); + ADD(_SHIFTER); + ADD(_ZERO); + ADD(_SC_1); + ADD(_SC_2); + ADD(_SC_3); + ADD(_SC_4); + ADD(_PI_4); + ADD(((address)_PI_4+8)); + ADD(_PI32INV); + ADD(_NEG_ZERO); + ADD(_P_1); + ADD(_P_2); + ADD(_P_3); + ADD(_PI_INV_TABLE); + ADD(_Ctable); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp index 9555d60c8a42c..1c88906f09aa6 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp @@ -1032,3 +1032,14 @@ void StubGenerator::generate_dilithium_stubs() { generate_dilithiumDecomposePoly_avx512(this, _masm); } } + +void StubGenerator::init_AOTAddressTable_dilithium(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)(addr)) + // use accessors to correctly identify the relevant addresses + ADD(dilithiumAvx512PermsAddr()); + ADD(dilithiumAvx512ConstsAddr(montQInvModRIdx)); + ADD(dilithiumAvx512ConstsAddr(dilithium_qIdx)); + ADD(dilithiumAvx512ConstsAddr(montRSquareModQIdx)); + ADD(dilithiumAvx512ConstsAddr(barrettAddendIdx)); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp index 5130fd2c9d2d4..32bbee73f5430 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp @@ -385,3 +385,22 @@ address StubGenerator::generate_libmExp() { } #undef __ + +void StubGenerator::init_AOTAddressTable_exp(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(_cv); + ADD(((address)_cv+16)); + ADD(((address)_cv+32)); + ADD(((address)_cv+48)); + ADD(((address)_cv+64)); + ADD(((address)_cv+80)); + ADD(_mmask); + ADD(_bias); + ADD(_Tbl_addr); + ADD(_ALLONES); + ADD(_ebias); + ADD(_XMAX); + ADD(_XMIN); + ADD(_INF); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp index b1eaa4b8031ad..cfbaf97a5aa32 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp @@ -525,3 +525,13 @@ address StubGenerator::generate_libmFmod() { } #undef __ + +void StubGenerator::init_AOTAddressTable_fmod(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(CONST_NaN); + ADD(CONST_1p260); + ADD(CONST_MAX); + ADD(CONST_INF); + ADD(CONST_e307); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp index 6f05b1ab5e6b9..4286088b45e68 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp @@ -538,3 +538,12 @@ void StubGenerator::generateHtbl_eight_blocks(Register htbl) { } #undef __ + +void StubGenerator::init_AOTAddressTable_ghash(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(GHASH_SHUFFLE_MASK); + ADD(GHASH_LONG_SWAP_MASK); + ADD(GHASH_BYTE_SWAP_MASK); + ADD(GHASH_POLYNOMIAL); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp index 3e5593322d5ea..c934d3c8ceb2a 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp @@ -950,3 +950,19 @@ void StubGenerator::generate_kyber_stubs() { } } } + +void StubGenerator::init_AOTAddressTable_kyber(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)(addr)) + // use accessors to correctly identify the relevant addresses + ADD(kyberAvx512NttPermsAddr()); + ADD(kyberAvx512InverseNttPermsAddr()); + ADD(kyberAvx512_nttMultPermsAddr()); + ADD(kyberAvx512_12To16PermsAddr()); + ADD(kyberAvx512ConstsAddr(qOffset)); + ADD(kyberAvx512ConstsAddr(qInvModROffset)); + ADD(kyberAvx512ConstsAddr(dimHalfInverseOffset)); + ADD(kyberAvx512ConstsAddr(barretMultiplierOffset)); + ADD(kyberAvx512ConstsAddr(montRSquareModqOffset)); + ADD(kyberAvx512ConstsAddr(f00Offset)); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp index 6b5b4d704e3e3..13da0591a592e 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp @@ -708,3 +708,23 @@ address StubGenerator::generate_libmLog10() { } #undef __ + +void StubGenerator::init_AOTAddressTable_log(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(_L_tbl); + ADD(_log2); + ADD(((address)_log2+8)); + ADD(_coeff); + ADD(((address)_coeff+16)); + ADD(((address)_coeff+32)); + ADD(_HIGHSIGMASK_log10); + ADD(_LOG10_E); + ADD(((address)_LOG10_E+8)); + ADD(_L_tbl_log10); + ADD(_log2_log10); + ADD(((address)_log2_log10+8)); + ADD(_coeff_log10); + ADD(((address)_coeff_log10+16)); + ADD(((address)_coeff_log10+32)); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp index c80b2d16181c6..2a4874a78108f 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp @@ -1695,3 +1695,12 @@ void StubGenerator::poly1305_msg_mul_reduce_vec4_avx2( __ vpaddq(A1, A1, YTMP2, Assembler::AVX_256bit); //Add medium 42-bit bits from new blocks to accumulator __ vpaddq(A1, A1, YTMP5, Assembler::AVX_256bit); } +#undef __ + +void StubGenerator::init_AOTAddressTable_poly1305(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(POLY1305_PAD_MSG); + ADD(POLY1305_MASK42); + ADD(POLY1305_MASK44); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp index c439e0b370faf..ebe2414486900 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp @@ -764,3 +764,16 @@ address StubGenerator::generate_intpoly_assign() { __ ret(0); return start; } +#undef __ + +void StubGenerator::init_AOTAddressTable_poly_mont(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + // use accessors to retrieve all correct addresses + ADD(shift_1L()); + ADD(shift_1R()); + ADD(p256_mask52()); + ADD(mask_limb5()); + ADD(modulus_p256()); + ADD(modulus_p256(1)); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp index 3c3df7e6ac4f4..581b0ff1ca976 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp @@ -1863,3 +1863,31 @@ address StubGenerator::generate_libmPow() { } #undef __ + +void StubGenerator::init_AOTAddressTable_pow(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(_HIGHSIGMASK); + ADD(_LOG2_E); + ADD(_HIGHMASK_Y); + ADD((address)_HIGHMASK_Y+8); + ADD(_T_exp); + ADD(_e_coeff); + ADD((address)_e_coeff+16); + ADD((address)_e_coeff+32); + ADD(_coeff_h); + ADD((address)_coeff_h+8); + ADD(_HIGHMASK_LOG_X); + ADD(_HALFMASK); + ADD(_coeff_pow); + ADD((address)_coeff_pow+16); + ADD((address)_coeff_pow+32); + ADD((address)_coeff_pow+48); + ADD((address)_coeff_pow+64); + ADD((address)_coeff_pow+80); + ADD(_L_tbl_pow); + ADD(_log2_pow); + ADD(_DOUBLE2); + ADD(_DOUBLE0); + ADD(_DOUBLE0DOT5); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp index f9d876f34f377..6b830f02e6c9b 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp @@ -508,3 +508,12 @@ void StubGenerator::generate_sha3_stubs() { generate_sha3_implCompress(StubId::stubgen_sha3_implCompressMB_id, this, _masm); } } + +#undef __ + +void StubGenerator::init_AOTAddressTable_sha3(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(round_constsAddr()); + ADD(permsAndRotsAddr()); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp index 5290e73758128..2b313d6c10fa6 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp @@ -649,3 +649,9 @@ address StubGenerator::generate_libmSin() { } #undef __ + +void StubGenerator::init_AOTAddressTable_sin(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(_ALL_ONES); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp index 86e4ac2017605..7f190068467e6 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp @@ -523,3 +523,23 @@ address StubGenerator::generate_libmSinh() { } #undef __ + +void StubGenerator::init_AOTAddressTable_sinh(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(_L2E); + ADD(_L2E + 8); + ADD(_HALFMASK); + ADD(_Shifter); + ADD(_cv); + ADD(_cv + 16); + ADD(_cv + 32); + ADD(_cv + 48); + ADD(_cv + 64); + ADD(_T2f); + ADD(_T2_neg_f); + ADD(_pv); + ADD(_pv + 16); + ADD(_pv + 32); + ADD(_MASK3); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp index 4f14414652c2d..04534ed50dadd 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp @@ -1029,3 +1029,24 @@ address StubGenerator::generate_libmTan() { } #undef __ + +void StubGenerator::init_AOTAddressTable_tan(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(_MUL16); + ADD(_sign_mask_tan); + ADD(_PI32INV_tan); + ADD(_P_1_tan); + ADD(_P_2_tan); + ADD(_P_3_tan); + ADD(_Ctable_tan); + ADD(_MASK_35_tan); + ADD(_Q_11_tan); + ADD(_Q_9_tan); + ADD(_Q_7_tan); + ADD(_Q_5_tan); + ADD(_Q_3_tan); + ADD(_PI_4_tan); + ADD(((address)_PI_4_tan+8)); + ADD(_QQ_2_tan); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp index dce4fbfc4550c..e0618ced9d820 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp @@ -499,3 +499,23 @@ address StubGenerator::generate_libmTanh() { } #undef __ + +void StubGenerator::init_AOTAddressTable_tanh(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(_L2E); + ADD(_L2E + 8); + ADD(_HALFMASK); + ADD(_ONEMASK); + ADD(_TWOMASK); + ADD(_Shifter); + ADD(_cv); + ADD(_cv + 16); + ADD(_cv + 32); + ADD(_T2_neg_f); + ADD(_pv); + ADD(_pv + 16); + ADD(_pv + 32); + ADD(_MASK3); + ADD(_RMASK); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.cpp b/src/hotspot/cpu/x86/stubRoutines_x86.cpp index ee9cea08e647c..0a9a1838b4cf6 100644 --- a/src/hotspot/cpu/x86/stubRoutines_x86.cpp +++ b/src/hotspot/cpu/x86/stubRoutines_x86.cpp @@ -411,3 +411,28 @@ ATTRIBUTE_ALIGNED(64) const julong StubRoutines::x86::_k512_W[] = 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL, }; + +// publish addresses of external data defined in this file which may +// be referenced from stub or code +void StubRoutines::x86::init_AOTAddressTable(GrowableArray
& external_addresses) { +#define ADD(addr) external_addresses.append((address)addr); + ADD(&_mxcsr_std); + ADD(&_mxcsr_rz); + ADD(crc_by128_masks_addr()); + ADD(crc_by128_masks_addr() + 16); + ADD(crc_by128_masks_addr() + 32); + ADD(_crc_table); + ADD(crc_by128_masks_avx512_addr()); + ADD(crc_by128_masks_avx512_addr() + 16); + ADD(crc_by128_masks_avx512_addr() + 32); + ADD(_crc_table_avx512); + ADD(_crc32c_table_avx512); + ADD(_shuf_table_crc32_avx512); + // n.b. call accessor for this one to ensure the table is generated + ADD(crc32c_table_addr()); + ADD(_arrays_hashcode_powers_of_31); + ADD(_k256); + ADD(_k256_W); + ADD(_k512_W); +#undef ADD +} diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.hpp b/src/hotspot/cpu/x86/stubRoutines_x86.hpp index 7d13c4f6e7ac9..f4d9cd6eda30f 100644 --- a/src/hotspot/cpu/x86/stubRoutines_x86.hpp +++ b/src/hotspot/cpu/x86/stubRoutines_x86.hpp @@ -112,6 +112,8 @@ class x86 { static address arrays_hashcode_powers_of_31() { return (address)_arrays_hashcode_powers_of_31; } static void generate_CRC32C_table(bool is_pclmulqdq_supported); + + static void init_AOTAddressTable(GrowableArray
& external_addresses); }; #endif // CPU_X86_STUBROUTINES_X86_HPP diff --git a/src/hotspot/cpu/zero/stubGenerator_zero.cpp b/src/hotspot/cpu/zero/stubGenerator_zero.cpp index 08cb173b50709..d46da4b3723ab 100644 --- a/src/hotspot/cpu/zero/stubGenerator_zero.cpp +++ b/src/hotspot/cpu/zero/stubGenerator_zero.cpp @@ -213,7 +213,7 @@ class StubGenerator: public StubCodeGenerator { } public: - StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) { + StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) : StubCodeGenerator(code, blob_id, stub_data) { switch(blob_id) { case BlobId::stubgen_preuniverse_id: generate_preuniverse_stubs(); @@ -237,8 +237,12 @@ class StubGenerator: public StubCodeGenerator { } }; -void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) { - StubGenerator g(code, blob_id); +// nothing to do for ZERO +void StubGenerator_AOTAddressTable_init() { +} + +void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) { + StubGenerator g(code, blob_id, stub_data); } EntryFrame *EntryFrame::build(const intptr_t* parameters, diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp index a4c956ff5bea1..e1867c7c4ab88 100644 --- a/src/hotspot/share/c1/c1_Runtime1.cpp +++ b/src/hotspot/share/c1/c1_Runtime1.cpp @@ -278,10 +278,6 @@ bool Runtime1::initialize(BufferBlob* blob) { if (!generate_blob_for(blob, id)) { return false; } - if (id == StubId::c1_forward_exception_id) { - // publish early c1 stubs at this point so later stubs can refer to them - AOTCodeCache::init_early_c1_table(); - } } // printing #ifndef PRODUCT diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp index 04776f4c16c95..7fb0464e2b927 100644 --- a/src/hotspot/share/code/aotCodeCache.cpp +++ b/src/hotspot/share/code/aotCodeCache.cpp @@ -32,9 +32,12 @@ #include "classfile/javaAssertions.hpp" #include "code/aotCodeCache.hpp" #include "code/codeCache.hpp" +#include "gc/shared/barrierSetNMethod.hpp" #include "gc/shared/gcConfig.hpp" #include "logging/logStream.hpp" #include "memory/memoryReserver.hpp" +#include "prims/jvmtiThreadState.hpp" +#include "prims/upcallLinker.hpp" #include "runtime/deoptimization.hpp" #include "runtime/flags/flagSetting.hpp" #include "runtime/globals_extension.hpp" @@ -153,10 +156,13 @@ static uint32_t encode_id(AOTCodeEntry::Kind kind, int id) { } else if (kind == AOTCodeEntry::C1Blob) { assert(StubInfo::is_c1(static_cast(id)), "not a c1 blob id %d", id); return id; - } else { - // kind must be AOTCodeEntry::C2Blob + } else if (kind == AOTCodeEntry::C2Blob) { assert(StubInfo::is_c2(static_cast(id)), "not a c2 blob id %d", id); return id; + } else { + // kind must be AOTCodeEntry::StubGenBlob + assert(StubInfo::is_stubgen(static_cast(id)), "not a stubgen blob id %d", id); + return id; } } @@ -182,7 +188,7 @@ void AOTCodeCache::initialize() { } // Disable stubs caching until JDK-8357398 is fixed. - FLAG_SET_ERGO(AOTStubCaching, false); + // FLAG_SET_ERGO(AOTStubCaching, false); if (VerifyOops) { // Disable AOT stubs caching when VerifyOops flag is on. @@ -278,6 +284,19 @@ bool AOTCodeCache::open_cache(bool is_dumping, bool is_using) { return true; } +// Called after continuations_init() when continuation stub callouts +// have been initialized +void AOTCodeCache::init3() { + if (opened_cache == nullptr) { + return; + } + // initialize external routines for continuations so we can save + // generated continuation blob that references them + AOTCodeAddressTable* table = opened_cache->_table; + assert(table != nullptr, "should be initialized already"); + table->init_extrs2(); +} + void AOTCodeCache::close() { if (is_on()) { delete _cache; // Free memory @@ -354,24 +373,59 @@ AOTCodeCache::AOTCodeCache(bool is_dumping, bool is_using) : _table = new AOTCodeAddressTable(); } -void AOTCodeCache::init_early_stubs_table() { +void AOTCodeCache::add_stub_entries(StubId stub_id, address start, GrowableArray
*entries, int begin_idx) { + EntryId entry_id = StubInfo::entry_base(stub_id); + add_stub_entry(entry_id, start); + // skip past first entry + entry_id = StubInfo::next_in_stub(stub_id, entry_id); + // now check for any more entries + int count = StubInfo::entry_count(stub_id) - 1; + assert(start != nullptr, "invalid start address for stub %s", StubInfo::name(stub_id)); + assert(entries == nullptr || begin_idx + count <= entries->length(), "sanity"); + // write any extra entries + for (int i = 0; i < count; i++) { + assert(entry_id != EntryId::NO_ENTRYID, "not enough entries for stub %s", StubInfo::name(stub_id)); + address a = entries->at(begin_idx + i); + add_stub_entry(entry_id, a); + entry_id = StubInfo::next_in_stub(stub_id, entry_id); + } + assert(entry_id == EntryId::NO_ENTRYID, "too many entries for stub %s", StubInfo::name(stub_id)); +} + +void AOTCodeCache::add_stub_entry(EntryId entry_id, address a) { + if (a != nullptr) { + if (_table != nullptr) { + log_trace(aot, codecache, stubs)("Publishing stub entry %s at address " INTPTR_FORMAT, StubInfo::name(entry_id), p2i(a)); + return _table->add_stub_entry(entry_id, a); + } + } +} + +void AOTCodeCache::set_shared_stubs_complete() { + AOTCodeAddressTable* table = addr_table(); + if (table != nullptr) { + table->set_shared_stubs_complete(); + } +} + +void AOTCodeCache::set_c1_stubs_complete() { AOTCodeAddressTable* table = addr_table(); if (table != nullptr) { - table->init_early_stubs(); + table->set_c1_stubs_complete(); } } -void AOTCodeCache::init_shared_blobs_table() { +void AOTCodeCache::set_c2_stubs_complete() { AOTCodeAddressTable* table = addr_table(); if (table != nullptr) { - table->init_shared_blobs(); + table->set_c2_stubs_complete(); } } -void AOTCodeCache::init_early_c1_table() { +void AOTCodeCache::set_stubgen_stubs_complete() { AOTCodeAddressTable* table = addr_table(); if (table != nullptr) { - table->init_early_c1(); + table->set_stubgen_stubs_complete(); } } @@ -718,6 +772,7 @@ bool AOTCodeCache::finish_write() { AOTCodeEntry* entries_address = _store_entries; // Pointer to latest entry uint adapters_count = 0; uint shared_blobs_count = 0; + uint stubgen_blobs_count = 0; uint C1_blobs_count = 0; uint C2_blobs_count = 0; uint max_size = 0; @@ -745,6 +800,8 @@ bool AOTCodeCache::finish_write() { adapters_count++; } else if (kind == AOTCodeEntry::SharedBlob) { shared_blobs_count++; + } else if (kind == AOTCodeEntry::StubGenBlob) { + stubgen_blobs_count++; } else if (kind == AOTCodeEntry::C1Blob) { C1_blobs_count++; } else if (kind == AOTCodeEntry::C2Blob) { @@ -781,6 +838,7 @@ bool AOTCodeCache::finish_write() { log_debug(aot, codecache, exit)(" Adapters: total=%u", adapters_count); log_debug(aot, codecache, exit)(" Shared Blobs: total=%d", shared_blobs_count); + log_debug(aot, codecache, exit)(" StubGen Blobs: total=%d", stubgen_blobs_count); log_debug(aot, codecache, exit)(" C1 Blobs: total=%d", C1_blobs_count); log_debug(aot, codecache, exit)(" C2 Blobs: total=%d", C2_blobs_count); log_debug(aot, codecache, exit)(" AOT code cache size: %u bytes, max entry's size: %u bytes", size, max_size); @@ -790,7 +848,8 @@ bool AOTCodeCache::finish_write() { header->init(size, (uint)strings_count, strings_offset, entries_count, new_entries_offset, adapters_count, shared_blobs_count, - C1_blobs_count, C2_blobs_count); + stubgen_blobs_count, C1_blobs_count, + C2_blobs_count); log_info(aot, codecache, exit)("Wrote %d AOT code entries to AOT Code Cache", entries_count); } @@ -799,13 +858,42 @@ bool AOTCodeCache::finish_write() { //------------------Store/Load AOT code ---------------------- -bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name) { +bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name, AOTStubData* stub_data, CodeBuffer* code_buffer) { + assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind); + + // we only expect stub data and a code buffer for a multi stub blob + assert(AOTCodeEntry::is_multi_stub_blob(entry_kind) == (stub_data != nullptr), + "entry_kind %d does not match stub_data pointer %p", + entry_kind, stub_data); + + assert((stub_data == nullptr) == (code_buffer == nullptr), + "stub data and code buffer must both be null or both non null"); + + // If this is a stub and the cache is on for either load or dump we + // need to insert the stub entries into the AOTCacheAddressTable so + // that relocs which refer to entries defined by this blob get + // translated correctly. + // + // Entry insertion needs to be be done up front before writing the + // blob because some blobs rely on internal daisy-chain references + // from one entry to another. + // + // Entry insertion also needs to be done even if the cache is open + // for use but not for dump. This may be needed when an archived + // blob omits some entries -- either because of a config change or a + // load failure -- with the result that the entries end up being + // generated. These generated entry addresses may be needed to + // resolve references from subsequently loaded blobs (for either + // stubs or nmethods). + + if (is_on() && AOTCodeEntry::is_blob(entry_kind)) { + publish_stub_addresses(blob, (BlobId)id, stub_data); + } + AOTCodeCache* cache = open_for_dump(); if (cache == nullptr) { return false; } - assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind); - if (AOTCodeEntry::is_adapter(entry_kind) && !is_dumping_adapter()) { return false; } @@ -851,8 +939,44 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind } CodeBlob::archive_blob(&blob, archive_buffer); - uint reloc_data_size = blob.relocation_size(); - n = cache->write_bytes((address)blob.relocation_begin(), reloc_data_size); + // For a relocatable code blob its relocations are linked from the + // blob. However, for a non-relocatable (stubgen) blob we only have + // transient relocations attached to the code buffer that are added + // in order to support AOT-load time patching. in either case, we + // need to explicitly save these relocs when storing the blob to the + // archive so we can then reload them and reattach them to either + // the blob or to a code buffer when we reload the blob into a + // production JVM. + // + // Either way we are then in a position to iterate over the relocs + // and AOT patch the ones that refer to code that may move between + // assembly and production time. We also need to save and restore + // AOT address table indexes for the target addresses of affected + // relocs. That happens below. + + int reloc_count; + address reloc_data; + if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) { + CodeSection* cs = code_buffer->code_section(CodeBuffer::SECT_INSTS); + reloc_count = (cs->has_locs() ? cs->locs_count() : 0); + reloc_data = (reloc_count > 0 ? (address)cs->locs_start() : nullptr); + } else { + reloc_count = blob.relocation_size() / sizeof(relocInfo); + reloc_data = (address)blob.relocation_begin(); + } + n = cache->write_bytes(&reloc_count, sizeof(int)); + if (n != sizeof(int)) { + return false; + } + if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) { + // align to heap word size before writing the relocs so we can + // install them into a code buffer when they get restored + if (!cache->align_write()) { + return false; + } + } + uint reloc_data_size = (uint)(reloc_count * sizeof(relocInfo)); + n = cache->write_bytes(reloc_data, reloc_data_size); if (n != reloc_data_size) { return false; } @@ -875,7 +999,29 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind } #endif /* PRODUCT */ - if (!cache->write_relocations(blob)) { + // In the case of a multi-stub blob we need to write start, end, + // secondary entries and extras. For any other blob entry addresses + // beyond the blob start will be stored in the blob as offsets. + if (stub_data != nullptr) { + if (!cache->write_stub_data(blob, stub_data)) { + return false; + } + } + + // now we have added all the other data we can write the AOT + // relocations + + bool write_ok; + if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) { + CodeSection* cs = code_buffer->code_section(CodeBuffer::SECT_INSTS); + RelocIterator iter(cs); + write_ok = cache->write_relocations(blob, iter); + } else { + RelocIterator iter(&blob); + write_ok = cache->write_relocations(blob, iter); + } + + if (!write_ok) { if (!cache->failed()) { // We may miss an address in AOT table - skip this code blob. cache->set_write_position(entry_position); @@ -884,6 +1030,7 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind } uint entry_size = cache->_write_position - entry_position; + AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_kind, encode_id(entry_kind, id), entry_position, entry_size, name_offset, name_size, blob_offset, has_oop_maps, blob.content_begin()); @@ -891,19 +1038,130 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind return true; } +bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name) { + assert(!AOTCodeEntry::is_blob(entry_kind), + "wrong entry kind for numeric id %d", id); + return store_code_blob(blob, entry_kind, (uint)id, name, nullptr, nullptr); +} + bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id) { - assert(AOTCodeEntry::is_blob(entry_kind), + assert(AOTCodeEntry::is_single_stub_blob(entry_kind), "wrong entry kind for blob id %s", StubInfo::name(id)); - return store_code_blob(blob, entry_kind, (uint)id, StubInfo::name(id)); + return store_code_blob(blob, entry_kind, (uint)id, StubInfo::name(id), nullptr, nullptr); } -CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name) { +bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id, AOTStubData* stub_data, CodeBuffer* code_buffer) { + assert(AOTCodeEntry::is_multi_stub_blob(entry_kind), + "wrong entry kind for multi stub blob id %s", StubInfo::name(id)); + return store_code_blob(blob, entry_kind, (uint)id, StubInfo::name(id), stub_data, code_buffer); +} + +bool AOTCodeCache::write_stub_data(CodeBlob &blob, AOTStubData *stub_data) { + BlobId blob_id = stub_data->blob_id(); + StubId stub_id = StubInfo::stub_base(blob_id); + address blob_base = blob.code_begin(); + int stub_cnt = StubInfo::stub_count(blob_id); + int n; + + LogStreamHandle(Trace, aot, codecache, stubs) log; + + if (log.is_enabled()) { + log.print_cr("======== Stub data starts at offset %d", _write_position); + } + + for (int i = 0; i < stub_cnt; i++, stub_id = StubInfo::next_in_blob(blob_id, stub_id)) { + // for each stub we find in the ranges list we write an int + // sequence where + // + // - start_pos is the stub start address encoded as a code section offset + // + // - end is the stub end address encoded as an offset from start + // + // - N counts the number of stub-local entries/extras + // + // - offseti is a stub-local entry/extra address encoded as len for + // a null address otherwise as an offset in range [1,len-1] + + StubAddrRange& range = stub_data->get_range(i); + GrowableArray
& addresses = stub_data->address_array(); + int base = range.start_index(); + if (base >= 0) { + n = write_bytes(&stub_id, sizeof(StubId)); + if (n != sizeof(StubId)) { + return false; + } + address start = addresses.at(base); + assert (blob_base <= start, "sanity"); + uint offset = (uint)(start - blob_base); + n = write_bytes(&offset, sizeof(uint)); + if (n != sizeof(int)) { + return false; + } + address end = addresses.at(base + 1); + assert (start < end, "sanity"); + offset = (uint)(end - start); + n = write_bytes(&offset, sizeof(uint)); + if (n != sizeof(int)) { + return false; + } + // write number of secondary and extra entries + int count = range.count() - 2; + n = write_bytes(&count, sizeof(int)); + if (n != sizeof(int)) { + return false; + } + for (int j = 0; j < count; j++) { + address next = addresses.at(base + 2 + j); + if (next != nullptr) { + // n.b. This maps next == end to the stub length which + // means we will reconstitute the address as nullptr. That + // happens when we have a handler range covers the end of + // a stub and needs to be handled specially by the client + // that restores the extras. + assert(start <= next && next <= end, "sanity"); + offset = (uint)(next - start); + } else { + // this can happen when a stub is not generated or an + // extra is the common handler target + offset = (uint)(end - start); + } + n = write_bytes(&offset, sizeof(uint)); + if (n != sizeof(int)) { + return false; + } + } + if (log.is_enabled()) { + log.print_cr("======== wrote stub %s and %d addresses up to offset %d", + StubInfo::name(stub_id), range.count(), _write_position); + } + } + } + // we should have exhausted all stub ids in the blob + assert(stub_id == StubId::NO_STUBID, "sanity"); + // write NO_STUBID as an end marker + n = write_bytes(&stub_id, sizeof(StubId)); + if (n != sizeof(StubId)) { + return false; + } + + if (log.is_enabled()) { + log.print_cr("======== Stub data ends at offset %d", _write_position); + } + + return true; +} + +CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name, AOTStubData* stub_data) { AOTCodeCache* cache = open_for_use(); if (cache == nullptr) { return nullptr; } assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind); + assert(AOTCodeEntry::is_multi_stub_blob(entry_kind) == (stub_data != nullptr), + "entry_kind %d does not match stub_data pointer %p", + entry_kind, stub_data); + if (AOTCodeEntry::is_adapter(entry_kind) && !is_using_adapter()) { return nullptr; } @@ -917,20 +1175,32 @@ CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, c return nullptr; } AOTCodeReader reader(cache, entry); - CodeBlob* blob = reader.compile_code_blob(name); + CodeBlob* blob = reader.compile_code_blob(name, entry_kind, id, stub_data); log_debug(aot, codecache, stubs)("%sRead blob '%s' (id=%u, kind=%s) from AOT Code Cache", (blob == nullptr? "Failed to " : ""), name, id, aot_code_entry_kind_name[entry_kind]); return blob; } +CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name) { + assert(!AOTCodeEntry::is_blob(entry_kind), + "wrong entry kind for numeric id %d", id); + return load_code_blob(entry_kind, (uint)id, name, nullptr); +} + CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, BlobId id) { - assert(AOTCodeEntry::is_blob(entry_kind), + assert(AOTCodeEntry::is_single_stub_blob(entry_kind), "wrong entry kind for blob id %s", StubInfo::name(id)); - return load_code_blob(entry_kind, (uint)id, StubInfo::name(id)); + return load_code_blob(entry_kind, (uint)id, StubInfo::name(id), nullptr); } -CodeBlob* AOTCodeReader::compile_code_blob(const char* name) { +CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, BlobId id, AOTStubData* stub_data) { + assert(AOTCodeEntry::is_multi_stub_blob(entry_kind), + "wrong entry kind for blob id %s", StubInfo::name(id)); + return load_code_blob(entry_kind, (uint)id, StubInfo::name(id), stub_data); +} + +CodeBlob* AOTCodeReader::compile_code_blob(const char* name, AOTCodeEntry::Kind entry_kind, int id, AOTStubData* stub_data) { uint entry_position = _entry->offset(); // Read name @@ -945,13 +1215,19 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name) { return nullptr; } - // Read archived code blob + // Read archived code blob and related info uint offset = entry_position + _entry->blob_offset(); CodeBlob* archived_blob = (CodeBlob*)addr(offset); offset += archived_blob->size(); + int reloc_count = *(int*)addr(offset); offset += sizeof(int); + if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) { + // position of relocs will have been aligned to heap word size so + // we can install them into a code buffer + offset = align_up(offset, DATA_ALIGNMENT); + } address reloc_data = (address)addr(offset); - offset += archived_blob->relocation_size(); + offset += reloc_count * sizeof(relocInfo); set_read_position(offset); ImmutableOopMapSet* oop_maps = nullptr; @@ -959,11 +1235,13 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name) { oop_maps = read_oop_map_set(); } + // Note that for a non-relocatable blob reloc_data will not be + // restored into the blob. We fix that later. + CodeBlob* code_blob = CodeBlob::create(archived_blob, stored_name, reloc_data, - oop_maps - ); + oop_maps); if (code_blob == nullptr) { // no space left in CodeCache return nullptr; } @@ -975,7 +1253,49 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name) { read_dbg_strings(code_blob->dbg_strings()); #endif // PRODUCT - fix_relocations(code_blob); + if (AOTCodeEntry::is_blob(entry_kind)) { + BlobId blob_id = static_cast(id); + if (StubInfo::is_stubgen(blob_id)) { + assert(stub_data != nullptr, "sanity"); + read_stub_data(code_blob, stub_data); + } + // publish entries found either in stub_data or as offsets in blob + AOTCodeCache::publish_stub_addresses(*code_blob, blob_id, stub_data); + } + + // Now that all the entry points are in the address table we can + // read all the extra reloc info and fix up any addresses that need + // patching to adjust for a new location in a new JVM. We can be + // sure to correctly update all runtime references, including + // cross-linked stubs that are internally daisy-chained. If + // relocation fails and we have to re-generate any of the stubs then + // the entry points for newly generated stubs will get updated, + // ensuring that any other stubs or nmethods we need to relocate + // will use the correct address. + + // if we have a relocatable code blob then the relocs are already + // attached to the blob and we can iterate over it to find the ones + // we need to patch. With a non-relocatable code blob we need to + // wrap it with a CodeBuffer and then reattach the relocs to the + // code buffer. + + if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) { + // the blob doesn't have any proper runtime relocs but we can + // reinstate the AOT-load time relocs we saved from the code + // buffer that generated this blob in a new code buffer and use + // the latter to iterate over them + CodeBuffer code_buffer(code_blob); + relocInfo* locs = (relocInfo*)reloc_data; + code_buffer.insts()->initialize_shared_locs(locs, reloc_count); + code_buffer.insts()->set_locs_end(locs + reloc_count); + CodeSection *cs = code_buffer.code_section(CodeBuffer::SECT_INSTS); + RelocIterator reloc_iter(cs); + fix_relocations(code_blob, reloc_iter); + } else { + // the AOT-load time relocs will be in the blob's restored relocs + RelocIterator reloc_iter(code_blob); + fix_relocations(code_blob, reloc_iter); + } #ifdef ASSERT LogStreamHandle(Trace, aot, codecache, stubs) log; @@ -987,15 +1307,146 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name) { return code_blob; } -// ------------ process code and data -------------- +void AOTCodeReader::read_stub_data(CodeBlob* code_blob, AOTStubData* stub_data) { + GrowableArray
& addresses = stub_data->address_array(); + // Read the list of stub ids and associated start, end, secondary + // and extra addresses and install them in the stub data. + // + // Also insert all start and secondary addresses into the AOTCache + // address table so we correctly relocate this blob and any followng + // blobs/nmethods. + // + // n.b. if an error occurs and we need to regenerate any of these + // stubs the address table will be updated as a side-effect of + // regeneration. + + address blob_base = code_blob->code_begin(); + uint blob_size = (uint)(code_blob->code_end() - blob_base); + int offset = read_position(); + LogStreamHandle(Trace, aot, codecache, stubs) log; + if (log.is_enabled()) { + log.print_cr("======== Stub data starts at offset %d", offset); + } + // read stub and entries until we see NO_STUBID + StubId stub_id = *(StubId*)addr(offset); offset += sizeof(StubId); + // we ought to have at least one saved stub in the blob + assert(stub_id != StubId::NO_STUBID, "blob %s contains no stubs!", StubInfo::name(stub_data->blob_id())); + while (stub_id != StubId::NO_STUBID) { + assert(StubInfo::blob(stub_id) == stub_data->blob_id(), "sanity"); + int idx = StubInfo::stubgen_offset_in_blob(stub_data->blob_id(), stub_id); + StubAddrRange& range = stub_data->get_range(idx); + // we should only see a stub once + assert(range.start_index() < 0, "repeated entry for stub %s", StubInfo::name(stub_id)); + int address_base = addresses.length(); + // start is an offset from the blob base + uint start = *(uint*)addr(offset); offset += sizeof(uint); + assert(start < blob_size, "stub %s start offset %d exceeds buffer length %d", StubInfo::name(stub_id), start, blob_size); + address stub_start = blob_base + start; + addresses.append(stub_start); + // end is an offset from the stub start + uint end = *(uint*)addr(offset); offset += sizeof(uint); + assert(start + end <= blob_size, "stub %s end offset %d exceeds remaining buffer length %d", StubInfo::name(stub_id), end, blob_size - start); + addresses.append(stub_start + end); + // read count of secondary entries plus extras + int entries_count = *(int*)addr(offset); offset += sizeof(int); + assert(entries_count >= (StubInfo::entry_count(stub_id) - 1), "not enough entries for %s", StubInfo::name(stub_id)); + for (int i = 0; i < entries_count; i++) { + // entry offset is an offset from the stub start less than or + // equal to end + uint entry = *(uint*)addr(offset); offset += sizeof(uint); + assert(entry <= end, "stub %s entry offset %d lies beyond stub end %d", StubInfo::name(stub_id), entry, end); + if (entry < end) { + addresses.append(stub_start + entry); + } else { + // entry offset == end encodes a nullptr + addresses.append(nullptr); + } + } + if (log.is_enabled()) { + log.print_cr("======== read stub %s and %d addresses up to offset %d", + StubInfo::name(stub_id), 2 + entries_count, offset); + } + range.init_entry(address_base, 2 + entries_count); + // move on to next stub or NO_STUBID + stub_id = *(StubId*)addr(offset); offset += sizeof(StubId); + } + if (log.is_enabled()) { + log.print_cr("======== Stub data ends at offset %d", offset); + } + + set_read_position(offset); +} + +void AOTCodeCache::publish_external_addresses(GrowableArray
& addresses) { + DEBUG_ONLY( _passed_init2 = true; ) + if (opened_cache == nullptr) { + return; + } + + cache()->_table->add_external_addresses(addresses); +} + +void AOTCodeCache::publish_stub_addresses(CodeBlob &code_blob, BlobId blob_id, AOTStubData *stub_data) { + if (stub_data != nullptr) { + // register all entries in stub + assert(StubInfo::stub_count(blob_id) > 1, + "multiple stub data provided for single stub blob %s", + StubInfo::name(blob_id)); + assert(blob_id == stub_data->blob_id(), + "blob id %s does not match id in stub data %s", + StubInfo::name(blob_id), + StubInfo::name(stub_data->blob_id())); + // iterate over all stubs in the blob + StubId stub_id = StubInfo::stub_base(blob_id); + int stub_cnt = StubInfo::stub_count(blob_id); + GrowableArray
& addresses = stub_data->address_array(); + for (int i = 0; i < stub_cnt; i++) { + assert(stub_id != StubId::NO_STUBID, "sanity"); + StubAddrRange& range = stub_data->get_range(i); + int base = range.start_index(); + if (base >= 0) { + cache()->add_stub_entries(stub_id, addresses.at(base), &addresses, base + 2); + } + stub_id = StubInfo::next_in_blob(blob_id, stub_id); + } + // we should have exhausted all stub ids in the blob + assert(stub_id == StubId::NO_STUBID, "sanity"); + } else { + // register entry or entries for a single stub blob + StubId stub_id = StubInfo::stub_base(blob_id); + assert(StubInfo::stub_count(blob_id) == 1, + "multiple stub blob %s provided without stub data", + StubInfo::name(blob_id)); + address start = code_blob.code_begin(); + if (StubInfo::entry_count(stub_id) == 1) { + assert(!code_blob.is_deoptimization_stub(), "expecting multiple entries for stub %s", StubInfo::name(stub_id)); + // register the blob base address as the only entry + cache()->add_stub_entries(stub_id, start); + } else { + assert(code_blob.is_deoptimization_stub(), "only expecting one entry for stub %s", StubInfo::name(stub_id)); + DeoptimizationBlob *deopt_blob = code_blob.as_deoptimization_blob(); + assert(deopt_blob->unpack() == start, "unexpected offset 0x%x for deopt stub entry", (int)(deopt_blob->unpack() - start)); + GrowableArray
addresses; + addresses.append(deopt_blob->unpack_with_exception()); + addresses.append(deopt_blob->unpack_with_reexecution()); + addresses.append(deopt_blob->unpack_with_exception_in_tls()); +#if INCLUDE_JVMCI + addresses.append(deopt_blob->uncommon_trap()); + addresses.append(deopt_blob->implicit_exception_uncommon_trap()); +#endif // INCLUDE_JVMCI + cache()->add_stub_entries(stub_id, start, &addresses, 0); + } + } +} + + // ------------ process code and data -------------- // Can't use -1. It is valid value for jump to iteself destination // used by static call stub: see NativeJump::jump_destination(). #define BAD_ADDRESS_ID -2 -bool AOTCodeCache::write_relocations(CodeBlob& code_blob) { +bool AOTCodeCache::write_relocations(CodeBlob& code_blob, RelocIterator& iter) { GrowableArray reloc_data; - RelocIterator iter(&code_blob); LogStreamHandle(Trace, aot, codecache, reloc) log; while (iter.next()) { int idx = reloc_data.append(0); // default value @@ -1049,6 +1500,11 @@ bool AOTCodeCache::write_relocations(CodeBlob& code_blob) { // Write the count first int count = reloc_data.length(); write_bytes(&count, sizeof(int)); + if (log.is_enabled()) { + log.print_cr("======== extra relocations count=%d", count); + log.print( " {"); + } + bool first = true; for (GrowableArrayIterator iter = reloc_data.begin(); iter != reloc_data.end(); ++iter) { uint value = *iter; @@ -1056,23 +1512,43 @@ bool AOTCodeCache::write_relocations(CodeBlob& code_blob) { if (n != sizeof(uint)) { return false; } + if (log.is_enabled()) { + if (first) { + first = false; + log.print("%d", value); + } else { + log.print(", %d", value); + } + } } + log.print_cr("}"); return true; } -void AOTCodeReader::fix_relocations(CodeBlob* code_blob) { - LogStreamHandle(Trace, aot, reloc) log; +void AOTCodeReader::fix_relocations(CodeBlob *code_blob, RelocIterator& iter) { uint offset = read_position(); - int count = *(int*)addr(offset); + int reloc_count = *(int*)addr(offset); offset += sizeof(int); - if (log.is_enabled()) { - log.print_cr("======== extra relocations count=%d", count); - } uint* reloc_data = (uint*)addr(offset); - offset += (count * sizeof(uint)); + offset += (reloc_count * sizeof(uint)); set_read_position(offset); - RelocIterator iter(code_blob); + LogStreamHandle(Trace, aot, codecache, reloc) log; + if (log.is_enabled()) { + log.print_cr("======== extra relocations count=%d", reloc_count); + } + if (log.is_enabled()) { + log.print(" {"); + for(int i = 0; i < reloc_count; i++) { + if (i == 0) { + log.print("%d", reloc_data[i]); + } else { + log.print(", %d", reloc_data[i]); + } + } + log.print_cr("}"); + } + int j = 0; while (iter.next()) { switch (iter.type()) { @@ -1121,7 +1597,7 @@ void AOTCodeReader::fix_relocations(CodeBlob* code_blob) { } j++; } - assert(j == count, "sanity"); + assert(j == reloc_count, "sanity"); } bool AOTCodeCache::write_oop_map_set(CodeBlob& cb) { @@ -1231,27 +1707,23 @@ void AOTCodeReader::read_dbg_strings(DbgStrings& dbg_strings) { //======================= AOTCodeAddressTable =============== -// address table ids for generated routines, external addresses and C -// string addresses are partitioned into positive integer ranges -// defined by the following positive base and max values -// i.e. [_extrs_base, _extrs_base + _extrs_max -1], -// [_blobs_base, _blobs_base + _blobs_max -1], -// ... -// [_c_str_base, _c_str_base + _c_str_max -1], - -#define _extrs_max 100 -#define _stubs_max 3 +// address table ids for generated routine entry adresses, external +// addresses and C string addresses are partitioned into positive +// integer ranges defined by the following positive base and max +// values i.e. [_extrs_base, _extrs_base + _extrs_max -1], +// [_stubs_base, _stubs_base + _stubs_max -1], [_c_str_base, +// _c_str_base + _c_str_max -1], -#define _shared_blobs_max 20 -#define _C1_blobs_max 10 -#define _blobs_max (_shared_blobs_max+_C1_blobs_max) -#define _all_max (_extrs_max+_stubs_max+_blobs_max) +#define _extrs_max 350 +#define _stubs_max static_cast(EntryId::NUM_ENTRYIDS) #define _extrs_base 0 #define _stubs_base (_extrs_base + _extrs_max) -#define _shared_blobs_base (_stubs_base + _stubs_max) -#define _C1_blobs_base (_shared_blobs_base + _shared_blobs_max) -#define _blobs_end (_shared_blobs_base + _blobs_max) +#define _all_max (_stubs_base + _stubs_max) + +// setter for external addresses and string addresses inserts new +// addresses in the order they are encountered them which must remain +// th esame across an assembly run and subsequent production run #define SET_ADDRESS(type, addr) \ { \ @@ -1259,18 +1731,34 @@ void AOTCodeReader::read_dbg_strings(DbgStrings& dbg_strings) { assert(type##_length <= type##_max, "increase size"); \ } +// setter for stub entry addresses inserts them using the stub entry +// id as an index + +#define SET_ENTRY_ADDRESS(type, addr, entry_id) \ + { \ + int idx = static_cast(entry); \ + _stub_addr[idx] = (address) (addr); \ + } + static bool initializing_extrs = false; void AOTCodeAddressTable::init_extrs() { if (_extrs_complete || initializing_extrs) return; // Done already - assert(_blobs_end <= _all_max, "AOTCodeAddress table ranges need adjusting"); - initializing_extrs = true; _extrs_addr = NEW_C_HEAP_ARRAY(address, _extrs_max, mtCode); _extrs_length = 0; + { + // Required by initial stubs + SET_ADDRESS(_extrs, SharedRuntime::exception_handler_for_return_address); // used by forward_exception + SET_ADDRESS(_extrs, CompressedOops::base_addr()); // used by call_stub + SET_ADDRESS(_extrs, Thread::current); // used by call_stub + SET_ADDRESS(_extrs, SharedRuntime::throw_StackOverflowError); + SET_ADDRESS(_extrs, SharedRuntime::throw_delayed_StackOverflowError); + } + // Record addresses of VM runtime methods SET_ADDRESS(_extrs, SharedRuntime::fixup_callers_callsite); SET_ADDRESS(_extrs, SharedRuntime::handle_wrong_method); @@ -1279,6 +1767,79 @@ void AOTCodeAddressTable::init_extrs() { #if defined(AARCH64) && !defined(ZERO) SET_ADDRESS(_extrs, JavaThread::aarch64_get_thread_helper); #endif + +#ifndef PRODUCT + SET_ADDRESS(_extrs, &SharedRuntime::_jbyte_array_copy_ctr); // used by arraycopy stub on arm32 and x86_64 + SET_ADDRESS(_extrs, &SharedRuntime::_jshort_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_jint_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_jlong_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_oop_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_checkcast_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_unsafe_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_generic_array_copy_ctr); // used by arraycopy stub + SET_ADDRESS(_extrs, &SharedRuntime::_unsafe_set_memory_ctr); // used by arraycopy stub +#endif /* PRODUCT */ + + SET_ADDRESS(_extrs, SharedRuntime::enable_stack_reserved_zone); + +#if defined(AMD64) && !defined(ZERO) + SET_ADDRESS(_extrs, SharedRuntime::montgomery_multiply); + SET_ADDRESS(_extrs, SharedRuntime::montgomery_square); +#endif // defined(AMD64) && !defined(ZERO) + + SET_ADDRESS(_extrs, SharedRuntime::d2f); + SET_ADDRESS(_extrs, SharedRuntime::d2i); + SET_ADDRESS(_extrs, SharedRuntime::d2l); + SET_ADDRESS(_extrs, SharedRuntime::dcos); + SET_ADDRESS(_extrs, SharedRuntime::dexp); + SET_ADDRESS(_extrs, SharedRuntime::dlog); + SET_ADDRESS(_extrs, SharedRuntime::dlog10); + SET_ADDRESS(_extrs, SharedRuntime::dpow); +#ifndef ZERO + SET_ADDRESS(_extrs, SharedRuntime::drem); +#endif + SET_ADDRESS(_extrs, SharedRuntime::dsin); + SET_ADDRESS(_extrs, SharedRuntime::dtan); + SET_ADDRESS(_extrs, SharedRuntime::f2i); + SET_ADDRESS(_extrs, SharedRuntime::f2l); +#ifndef ZERO + SET_ADDRESS(_extrs, SharedRuntime::frem); +#endif + SET_ADDRESS(_extrs, SharedRuntime::l2d); + SET_ADDRESS(_extrs, SharedRuntime::l2f); + SET_ADDRESS(_extrs, SharedRuntime::ldiv); + SET_ADDRESS(_extrs, SharedRuntime::lmul); + SET_ADDRESS(_extrs, SharedRuntime::lrem); + +#if INCLUDE_JVMTI + SET_ADDRESS(_extrs, &JvmtiExport::_should_notify_object_alloc); +#endif /* INCLUDE_JVMTI */ + + SET_ADDRESS(_extrs, ThreadIdentifier::unsafe_offset()); + SET_ADDRESS(_extrs, Thread::current); + + SET_ADDRESS(_extrs, os::javaTimeMillis); + SET_ADDRESS(_extrs, os::javaTimeNanos); +#ifndef PRODUCT + SET_ADDRESS(_extrs, os::breakpoint); +#endif + +#if INCLUDE_JVMTI + SET_ADDRESS(_extrs, &JvmtiVTMSTransitionDisabler::_VTMS_notify_jvmti_events); +#endif /* INCLUDE_JVMTI */ + SET_ADDRESS(_extrs, StubRoutines::crc_table_addr()); +#ifndef PRODUCT + SET_ADDRESS(_extrs, &SharedRuntime::_partial_subtype_ctr); + SET_ADDRESS(_extrs, JavaThread::verify_cross_modify_fence_failure); +#endif + +#if INCLUDE_JFR + SET_ADDRESS(_extrs, JfrIntrinsicSupport::write_checkpoint); + SET_ADDRESS(_extrs, JfrIntrinsicSupport::return_lease); +#endif + + SET_ADDRESS(_extrs, UpcallLinker::handle_uncaught_exception); // used by upcall_stub_exception_handler + { // Required by Shared blobs SET_ADDRESS(_extrs, Deoptimization::fetch_unroll_info); @@ -1287,7 +1848,6 @@ void AOTCodeAddressTable::init_extrs() { SET_ADDRESS(_extrs, SharedRuntime::resolve_opt_virtual_call_C); SET_ADDRESS(_extrs, SharedRuntime::resolve_virtual_call_C); SET_ADDRESS(_extrs, SharedRuntime::resolve_static_call_C); - SET_ADDRESS(_extrs, SharedRuntime::throw_StackOverflowError); SET_ADDRESS(_extrs, SharedRuntime::throw_delayed_StackOverflowError); SET_ADDRESS(_extrs, SharedRuntime::throw_AbstractMethodError); SET_ADDRESS(_extrs, SharedRuntime::throw_IncompatibleClassChangeError); @@ -1298,7 +1858,6 @@ void AOTCodeAddressTable::init_extrs() { { // Required by C1 blobs SET_ADDRESS(_extrs, static_cast(SharedRuntime::dtrace_object_alloc)); - SET_ADDRESS(_extrs, SharedRuntime::exception_handler_for_return_address); SET_ADDRESS(_extrs, SharedRuntime::register_finalizer); SET_ADDRESS(_extrs, Runtime1::is_instance_of); SET_ADDRESS(_extrs, Runtime1::exception_handler_for_pc); @@ -1315,7 +1874,6 @@ void AOTCodeAddressTable::init_extrs() { SET_ADDRESS(_extrs, Runtime1::throw_array_store_exception); SET_ADDRESS(_extrs, Runtime1::throw_class_cast_exception); SET_ADDRESS(_extrs, Runtime1::throw_incompatible_class_change_error); - SET_ADDRESS(_extrs, Runtime1::is_instance_of); SET_ADDRESS(_extrs, Runtime1::monitorenter); SET_ADDRESS(_extrs, Runtime1::monitorexit); SET_ADDRESS(_extrs, Runtime1::deoptimize); @@ -1327,9 +1885,6 @@ void AOTCodeAddressTable::init_extrs() { SET_ADDRESS(_extrs, Runtime1::unimplemented_entry); SET_ADDRESS(_extrs, Thread::current); SET_ADDRESS(_extrs, CompressedKlassPointers::base_addr()); -#ifndef PRODUCT - SET_ADDRESS(_extrs, os::breakpoint); -#endif } #endif @@ -1358,14 +1913,16 @@ void AOTCodeAddressTable::init_extrs() { SET_ADDRESS(_extrs, OptoRuntime::rethrow_C); SET_ADDRESS(_extrs, OptoRuntime::slow_arraycopy_C); SET_ADDRESS(_extrs, OptoRuntime::register_finalizer_C); -#if defined(AARCH64) - SET_ADDRESS(_extrs, JavaThread::verify_cross_modify_fence_failure); -#endif // AARCH64 } #endif // COMPILER2 #if INCLUDE_G1GC SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_field_pre_entry); + SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry); // used by arraycopy stubs + SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_array_pre_oop_entry); // used by arraycopy stubs + SET_ADDRESS(_extrs, G1BarrierSetRuntime::write_ref_array_post_entry); // used by arraycopy stubs + SET_ADDRESS(_extrs, BarrierSetNMethod::nmethod_stub_entry_barrier); // used by method_entry_barrier + #endif #if INCLUDE_SHENANDOAHGC SET_ADDRESS(_extrs, ShenandoahRuntime::write_barrier_pre); @@ -1384,91 +1941,70 @@ void AOTCodeAddressTable::init_extrs() { #endif #endif // ZERO - _extrs_complete = true; - log_debug(aot, codecache, init)("External addresses recorded"); + log_debug(aot, codecache, init)("External addresses opened and recorded"); + // allocate storage for stub entries + _stubs_addr = NEW_C_HEAP_ARRAY(address, _stubs_max, mtCode); + log_debug(aot, codecache, init)("Stub addresses opened"); } -static bool initializing_early_stubs = false; - -void AOTCodeAddressTable::init_early_stubs() { - if (_complete || initializing_early_stubs) return; // Done already - initializing_early_stubs = true; - _stubs_addr = NEW_C_HEAP_ARRAY(address, _stubs_max, mtCode); - _stubs_length = 0; - SET_ADDRESS(_stubs, StubRoutines::forward_exception_entry()); +void AOTCodeAddressTable::init_extrs2() { + assert(initializing_extrs && !_extrs_complete, + "invalid sequence for init_extrs2"); { - // Required by C1 blobs -#if defined(AMD64) && !defined(ZERO) - SET_ADDRESS(_stubs, StubRoutines::x86::double_sign_flip()); - SET_ADDRESS(_stubs, StubRoutines::x86::d2l_fixup()); -#endif // AMD64 + SET_ADDRESS(_extrs, Continuation::prepare_thaw); // used by cont_thaw + SET_ADDRESS(_extrs, Continuation::thaw_entry()); // used by cont_thaw + SET_ADDRESS(_extrs, ContinuationEntry::thaw_call_pc_address()); // used by cont_preempt_stub } - - _early_stubs_complete = true; - log_info(aot, codecache, init)("Early stubs recorded"); + _extrs_complete = true; + initializing_extrs = false; + log_debug(aot, codecache, init)("External addresses recorded and closed"); } -static bool initializing_shared_blobs = false; - -void AOTCodeAddressTable::init_shared_blobs() { - if (_complete || initializing_shared_blobs) return; // Done already - initializing_shared_blobs = true; - address* blobs_addr = NEW_C_HEAP_ARRAY(address, _blobs_max, mtCode); - - // Divide _shared_blobs_addr array to chunks because they could be initialized in parrallel - _shared_blobs_addr = blobs_addr; - _C1_blobs_addr = _shared_blobs_addr + _shared_blobs_max; - - _shared_blobs_length = 0; - _C1_blobs_length = 0; +void AOTCodeAddressTable::add_external_addresses(GrowableArray
& addresses) { + assert(initializing_extrs && !_extrs_complete, + "invalid sequence for add_external_addresses"); + for (int i = 0; i < addresses.length(); i++) { + SET_ADDRESS(_extrs, addresses.at(i)); + } + log_debug(aot, codecache, init)("External addresses recorded"); +} - // clear the address table - memset(blobs_addr, 0, sizeof(address)* _blobs_max); +void AOTCodeAddressTable::add_stub_entry(EntryId entry_id, address a) { + assert(_extrs_complete || initializing_extrs, + "recording stub entry address before external addresses complete"); + assert(!(StubInfo::is_shared(StubInfo::stub(entry_id)) && _shared_stubs_complete), "too late to add shared entry"); + assert(!(StubInfo::is_stubgen(StubInfo::stub(entry_id)) && _stubgen_stubs_complete), "too late to add stubgen entry"); + assert(!(StubInfo::is_c1(StubInfo::stub(entry_id)) && _c1_stubs_complete), "too late to add c1 entry"); + assert(!(StubInfo::is_c2(StubInfo::stub(entry_id)) && _c2_stubs_complete), "too late to add c2 entry"); + log_debug(aot, stubs)("Recording address 0x%p for %s entry %s", a, StubInfo::name(StubInfo::stubgroup(entry_id)), StubInfo::name(entry_id)); + int idx = static_cast(entry_id); + _stubs_addr[idx] = a; +} - // Record addresses of generated code blobs - SET_ADDRESS(_shared_blobs, SharedRuntime::get_handle_wrong_method_stub()); - SET_ADDRESS(_shared_blobs, SharedRuntime::get_ic_miss_stub()); - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack()); - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_exception()); - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_reexecution()); - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->unpack_with_exception_in_tls()); -#if INCLUDE_JVMCI - if (EnableJVMCI) { - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->uncommon_trap()); - SET_ADDRESS(_shared_blobs, SharedRuntime::deopt_blob()->implicit_exception_uncommon_trap()); - } -#endif +void AOTCodeAddressTable::set_shared_stubs_complete() { + assert(!_shared_stubs_complete, "repeated close for shared stubs!"); + _shared_stubs_complete = true; + log_debug(aot, codecache, init)("Shared stubs closed"); +} - _shared_blobs_complete = true; - log_debug(aot, codecache, init)("Early shared blobs recorded"); - _complete = true; +void AOTCodeAddressTable::set_c1_stubs_complete() { + assert(!_c1_stubs_complete, "repeated close for c1 stubs!"); + _c2_stubs_complete = true; + log_debug(aot, codecache, init)("C1 stubs closed"); } -void AOTCodeAddressTable::init_early_c1() { -#ifdef COMPILER1 - // Runtime1 Blobs - StubId id = StubInfo::stub_base(StubGroup::C1); - // include forward_exception in range we publish - StubId limit = StubInfo::next(StubId::c1_forward_exception_id); - for (; id != limit; id = StubInfo::next(id)) { - if (Runtime1::blob_for(id) == nullptr) { - log_info(aot, codecache, init)("C1 blob %s is missing", Runtime1::name_for(id)); - continue; - } - if (Runtime1::entry_for(id) == nullptr) { - log_info(aot, codecache, init)("C1 blob %s is missing entry", Runtime1::name_for(id)); - continue; - } - address entry = Runtime1::entry_for(id); - SET_ADDRESS(_C1_blobs, entry); - } -#endif // COMPILER1 - assert(_C1_blobs_length <= _C1_blobs_max, "increase _C1_blobs_max to %d", _C1_blobs_length); - _early_c1_complete = true; +void AOTCodeAddressTable::set_c2_stubs_complete() { + assert(!_c2_stubs_complete, "repeated close for c2 stubs!"); + _c2_stubs_complete = true; + log_debug(aot, codecache, init)("C2 stubs closed"); } -#undef SET_ADDRESS +void AOTCodeAddressTable::set_stubgen_stubs_complete() { + assert(!_stubgen_stubs_complete, "repeated close for stubgen stubs!"); + _stubgen_stubs_complete = true; + log_debug(aot, codecache, init)("StubGen stubs closed"); +} AOTCodeAddressTable::~AOTCodeAddressTable() { if (_extrs_addr != nullptr) { @@ -1477,9 +2013,6 @@ AOTCodeAddressTable::~AOTCodeAddressTable() { if (_stubs_addr != nullptr) { FREE_C_HEAP_ARRAY(address, _stubs_addr); } - if (_shared_blobs_addr != nullptr) { - FREE_C_HEAP_ARRAY(address, _shared_blobs_addr); - } } #ifdef PRODUCT @@ -1564,7 +2097,7 @@ const char* AOTCodeCache::add_C_string(const char* str) { } const char* AOTCodeAddressTable::add_C_string(const char* str) { - if (_extrs_complete) { + if (_extrs_complete || initializing_extrs) { // Check previous strings address for (int i = 0; i < _C_strings_count; i++) { if (_C_strings_in[i] == str) { @@ -1627,7 +2160,7 @@ static int search_address(address addr, address* table, uint length) { } address AOTCodeAddressTable::address_for_id(int idx) { - assert(_extrs_complete, "AOT Code Cache VM runtime addresses table is not complete"); + assert(_extrs_complete || initializing_extrs, "AOT Code Cache VM runtime addresses table is not complete"); if (idx == -1) { return (address)-1; } @@ -1644,15 +2177,9 @@ address AOTCodeAddressTable::address_for_id(int idx) { if (/* id >= _extrs_base && */ id < _extrs_length) { return _extrs_addr[id - _extrs_base]; } - if (id >= _stubs_base && id < _stubs_base + _stubs_length) { + if (id >= _stubs_base && id < _c_str_base) { return _stubs_addr[id - _stubs_base]; } - if (id >= _shared_blobs_base && id < _shared_blobs_base + _shared_blobs_length) { - return _shared_blobs_addr[id - _shared_blobs_base]; - } - if (id >= _C1_blobs_base && id < _C1_blobs_base + _C1_blobs_length) { - return _C1_blobs_addr[id - _C1_blobs_base]; - } if (id >= _c_str_base && id < (_c_str_base + (uint)_C_strings_count)) { return address_for_C_string(id - _c_str_base); } @@ -1661,7 +2188,7 @@ address AOTCodeAddressTable::address_for_id(int idx) { } int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeBlob* code_blob) { - assert(_extrs_complete, "AOT Code Cache VM runtime addresses table is not complete"); + assert(_extrs_complete || initializing_extrs, "AOT Code Cache VM runtime addresses table is not complete"); int id = -1; if (addr == (address)-1) { // Static call stub has jump to itself return id; @@ -1671,9 +2198,9 @@ int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeB if (id >= 0) { return id + _c_str_base; } - if (StubRoutines::contains(addr)) { - // Search in stubs - id = search_address(addr, _stubs_addr, _stubs_length); + if (StubRoutines::contains(addr) || CodeCache::find_blob(addr) != nullptr) { + // Search for a matching stub entry + id = search_address(addr, _stubs_addr, _stubs_max); if (id < 0) { StubCodeDesc* desc = StubCodeDesc::desc_for(addr); if (desc == nullptr) { @@ -1685,51 +2212,39 @@ int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeB return id + _stubs_base; } } else { - CodeBlob* cb = CodeCache::find_blob(addr); - if (cb != nullptr) { - // Search in code blobs - int id_base = _shared_blobs_base; - id = search_address(addr, _shared_blobs_addr, _blobs_max); - if (id < 0) { - assert(false, "Address " INTPTR_FORMAT " for Blob:%s is missing in AOT Code Cache addresses table", p2i(addr), cb->name()); - } else { - return id_base + id; - } - } else { - // Search in runtime functions - id = search_address(addr, _extrs_addr, _extrs_length); - if (id < 0) { - ResourceMark rm; - const int buflen = 1024; - char* func_name = NEW_RESOURCE_ARRAY(char, buflen); - int offset = 0; - if (os::dll_address_to_function_name(addr, func_name, buflen, &offset)) { - if (offset > 0) { - // Could be address of C string - uint dist = (uint)pointer_delta(addr, (address)os::init, 1); - log_debug(aot, codecache)("Address " INTPTR_FORMAT " (offset %d) for runtime target '%s' is missing in AOT Code Cache addresses table", - p2i(addr), dist, (const char*)addr); - assert(dist > (uint)(_all_max + MAX_STR_COUNT), "change encoding of distance"); - return dist; - } + // Search in runtime functions + id = search_address(addr, _extrs_addr, _extrs_length); + if (id < 0) { + ResourceMark rm; + const int buflen = 1024; + char* func_name = NEW_RESOURCE_ARRAY(char, buflen); + int offset = 0; + if (os::dll_address_to_function_name(addr, func_name, buflen, &offset)) { + if (offset > 0) { + // Could be address of C string + uint dist = (uint)pointer_delta(addr, (address)os::init, 1); + log_debug(aot, codecache)("Address " INTPTR_FORMAT " (offset %d) for runtime target '%s' is missing in AOT Code Cache addresses table", + p2i(addr), dist, (const char*)addr); + assert(dist > (uint)(_all_max + MAX_STR_COUNT), "change encoding of distance"); + return dist; + } #ifdef ASSERT - reloc.print_current_on(tty); - code_blob->print_on(tty); - code_blob->print_code_on(tty); - assert(false, "Address " INTPTR_FORMAT " for runtime target '%s+%d' is missing in AOT Code Cache addresses table", p2i(addr), func_name, offset); + reloc.print_current_on(tty); + code_blob->print_on(tty); + code_blob->print_code_on(tty); + assert(false, "Address " INTPTR_FORMAT " for runtime target '%s+%d' is missing in AOT Code Cache addresses table", p2i(addr), func_name, offset); #endif - } else { + } else { #ifdef ASSERT - reloc.print_current_on(tty); - code_blob->print_on(tty); - code_blob->print_code_on(tty); - os::find(addr, tty); - assert(false, "Address " INTPTR_FORMAT " for /('%s') is missing in AOT Code Cache addresses table", p2i(addr), (const char*)addr); + reloc.print_current_on(tty); + code_blob->print_on(tty); + code_blob->print_code_on(tty); + os::find(addr, tty); + assert(false, "Address " INTPTR_FORMAT " for /('%s') is missing in AOT Code Cache addresses table", p2i(addr), (const char*)addr); #endif - } - } else { - return _extrs_base + id; } + } else { + return _extrs_base + id; } } return id; @@ -1758,3 +2273,176 @@ void AOTCodeCache::print_on(outputStream* st) { } } } + +// methods for managing entries in multi-stub blobs + + +AOTStubData::AOTStubData(BlobId blob_id) : + _blob_id(blob_id), + _cached_blob(nullptr), + _stub_cnt(0), + _ranges(nullptr), + _current(StubId::NO_STUBID), + _current_idx(-1), + _flags(0) { + assert(StubInfo::is_stubgen(blob_id), + "AOTStubData expects a multi-stub blob not %s", + StubInfo::name(blob_id)); + + // we cannot save or restore preuniversestubs because the cache + // cannot be accessed before initialising the universe + if (blob_id == BlobId::stubgen_preuniverse_id) { + // invalidate any attempt to use this + _flags |= INVALID; + return; + } + if (AOTCodeCache::is_on()) { + // allow update of stub entry addresses + _flags |= OPEN; + if (AOTCodeCache::is_using_stub()) { + // allow stub loading + _flags |= USING; + } + if (AOTCodeCache::is_dumping_stub()) { + // allow stub saving + _flags |= DUMPING; + } + // we need to track all the blob's entries + _stub_cnt = StubInfo::stub_count(_blob_id); + _ranges = NEW_C_HEAP_ARRAY(StubAddrRange, _stub_cnt, mtCode); + for (int i = 0; i < _stub_cnt; i++) { + _ranges[i].default_init(); + } + } +} + +bool AOTStubData::load_code_blob() { + assert(is_using(), "should not call"); + assert(!is_invalid() && _cached_blob == nullptr, "repeated init"); + _cached_blob = AOTCodeCache::load_code_blob(AOTCodeEntry::StubGenBlob, + _blob_id, + this); + if (_cached_blob == nullptr) { + set_invalid(); + return false; + } else { + return true; + } +} + +bool AOTStubData::store_code_blob(CodeBlob& new_blob, CodeBuffer *code_buffer) { + assert(is_dumping(), "should not call"); + assert(_cached_blob == nullptr, "should not be loading and storing!"); + if (!AOTCodeCache::store_code_blob(new_blob, + AOTCodeEntry::StubGenBlob, + _blob_id, this, code_buffer)) { + set_invalid(); + return false; + } else { + return true; + } +} + +bool AOTStubData::find_archive_data(StubId stub_id) { + assert(StubInfo::blob(stub_id) == _blob_id, "sanity check"); + if (is_invalid()) { + return false; + } + int idx = StubInfo::stubgen_offset_in_blob(_blob_id, stub_id); + assert(idx >= 0 && idx < _stub_cnt, "invalid index %d for stub count %d", idx, _stub_cnt); + // ensure we have a valid associated range + StubAddrRange range = _ranges[idx]; + int start_index = range.start_index(); + if (start_index < 0) { + _current = StubId::NO_STUBID; +#ifdef DEBUG + // reset index so we can idenitfy which ones we failed to find + range.init_entry(-2, 0); +#endif + return false; + } + _current = stub_id; + _current_idx = idx; + return true; +} + +void AOTStubData::load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
* entries, GrowableArray
* extras) { + assert(StubInfo::blob(stub_id) == _blob_id, "sanity check"); + assert(_current == stub_id && stub_id != StubId::NO_STUBID, "sanity check"); + assert(!is_invalid(), "should not load stubs when archive data is invalid"); + assert(_current_idx >= 0, "sanity"); + StubAddrRange& range = _ranges[_current_idx]; + int base = range.start_index(); + int count = range.count(); + assert(base >= 0, "sanity"); + assert(count >= 2, "sanity"); + // first two saved addresses are start and end + start = _address_array.at(base); + end = _address_array.at(base + 1); + assert(start != nullptr, "failed to load start address of stub %s", StubInfo::name(stub_id)); + assert(end != nullptr, "failed to load end address of stub %s", StubInfo::name(stub_id)); + assert(start < end, "start address %p should be less than end %p address for stub %s", start, end, StubInfo::name(stub_id)); + + int entry_count = StubInfo::entry_count(stub_id); + // the address count must at least include the stub start, end + // and secondary addresses + assert(count >= entry_count + 1, "stub %s requires %d saved addresses but only has %d", StubInfo::name(stub_id), entry_count + 1, count); + + // caller must retrieve secondary entries if and only if they exist + assert((entry_count == 1) == (entries == nullptr), "trying to retrieve wrong number of entries for stub %s", StubInfo::name(stub_id)); + int index = 2; + if (entries != nullptr) { + assert(entries->length() == 0, "non-empty array when retrieving entries for stub %s!", StubInfo::name(stub_id)); + while (index < entry_count + 1) { + address entry = _address_array.at(base + index++); + assert(entry == nullptr || (start < entry && entry < end), "entry address %p not in range (%p, %p) for stub %s", entry, start, end, StubInfo::name(stub_id)); + entries->append(entry); + } + } + // caller must retrieve extras if and only if they exist + assert((index < count) == (extras != nullptr), "trying to retrieve wrong number of extras for stub %s", StubInfo::name(stub_id)); + if (extras != nullptr) { + assert(extras->length() == 0, "non-empty array when retrieving extras for stub %s!", StubInfo::name(stub_id)); + while (index < count) { + address extra = _address_array.at(base + index++); + assert(extra == nullptr || (start <= extra && extra < end), "extra address %p not in range (%p, %p) for stub %s", extra, start, end, StubInfo::name(stub_id)); + extras->append(extra); + } + } +} + +void AOTStubData::store_archive_data(StubId stub_id, address start, address end, GrowableArray
* entries, GrowableArray
* extras) { + assert(StubInfo::blob(stub_id) == _blob_id, "sanity check"); + assert(start != nullptr, "start address cannot be null"); + assert(end != nullptr, "end address cannot be null"); + assert(start < end, "start address %p should be less than end %p address for stub %s", start, end, StubInfo::name(stub_id)); + _current = stub_id; + _current_idx = StubInfo::stubgen_offset_in_blob(_blob_id, stub_id); + StubAddrRange& range = _ranges[_current_idx]; + assert(range.start_index() == -1, "sanity"); + int base = _address_array.length(); + assert(base >= 0, "sanity"); + // first two saved addresses are start and end + _address_array.append(start); + _address_array.append(end); + // caller must save secondary entries if and only if they exist + assert((StubInfo::entry_count(stub_id) == 1) == (entries == nullptr), "trying to save wrong number of entries for stub %s", StubInfo::name(stub_id)); + if (entries != nullptr) { + assert(entries->length() == StubInfo::entry_count(stub_id) - 1, "incorrect entry count %d when saving entries for stub %s!", entries->length(), StubInfo::name(stub_id)); + for (int i = 0; i < entries->length(); i++) { + address entry = entries->at(i); + assert(entry == nullptr || (start < entry && entry < end), "entry address %p not in range (%p, %p) for stub %s", entry, start, end, StubInfo::name(stub_id)); + _address_array.append(entry); + } + } + // caller may wish to save extra addresses + if (extras != nullptr) { + for (int i = 0; i < extras->length(); i++) { + address extra = extras->at(i); + // handler range end may be end -- it gets restored as nullptr + assert(extra == nullptr || (start <= extra && extra <= end), "extra address %p not in range (%p, %p) for stub %s", extra, start, end, StubInfo::name(stub_id)); + _address_array.append(extra); + } + } + range.init_entry(base, _address_array.length() - base); +} diff --git a/src/hotspot/share/code/aotCodeCache.hpp b/src/hotspot/share/code/aotCodeCache.hpp index 778ad34e44817..8b7ecfe088d07 100644 --- a/src/hotspot/share/code/aotCodeCache.hpp +++ b/src/hotspot/share/code/aotCodeCache.hpp @@ -37,6 +37,7 @@ class CodeBuffer; class RelocIterator; class AOTCodeCache; +class AOTCodeReader; class AdapterBlob; class ExceptionBlob; class ImmutableOopMapSet; @@ -52,6 +53,7 @@ enum CompLevel : signed char; Fn(SharedBlob) \ Fn(C1Blob) \ Fn(C2Blob) \ + Fn(StubGenBlob) \ // Descriptor of AOT Code Cache's entry class AOTCodeEntry { @@ -113,7 +115,9 @@ class AOTCodeEntry { address dumptime_content_start_addr() const { return _dumptime_content_start_addr; } static bool is_valid_entry_kind(Kind kind) { return kind > None && kind < Kind_count; } - static bool is_blob(Kind kind) { return kind == SharedBlob || kind == C1Blob || kind == C2Blob; } + static bool is_blob(Kind kind) { return kind == SharedBlob || kind == C1Blob || kind == C2Blob || kind == StubGenBlob; } + static bool is_single_stub_blob(Kind kind) { return kind == SharedBlob || kind == C1Blob || kind == C2Blob || kind == StubGenBlob; } + static bool is_multi_stub_blob(Kind kind) { return kind == StubGenBlob; } static bool is_adapter(Kind kind) { return kind == Adapter; } }; @@ -122,40 +126,36 @@ class AOTCodeAddressTable : public CHeapObj { private: address* _extrs_addr; address* _stubs_addr; - address* _shared_blobs_addr; - address* _C1_blobs_addr; uint _extrs_length; - uint _stubs_length; - uint _shared_blobs_length; - uint _C1_blobs_length; bool _extrs_complete; - bool _early_stubs_complete; - bool _shared_blobs_complete; - bool _early_c1_complete; + bool _shared_stubs_complete; + bool _c1_stubs_complete; + bool _c2_stubs_complete; + bool _stubgen_stubs_complete; bool _complete; public: AOTCodeAddressTable() : _extrs_addr(nullptr), _stubs_addr(nullptr), - _shared_blobs_addr(nullptr), - _C1_blobs_addr(nullptr), _extrs_length(0), - _stubs_length(0), - _shared_blobs_length(0), - _C1_blobs_length(0), _extrs_complete(false), - _early_stubs_complete(false), - _shared_blobs_complete(false), - _early_c1_complete(false), + _shared_stubs_complete(false), + _c1_stubs_complete(false), + _c2_stubs_complete(false), + _stubgen_stubs_complete(false), _complete(false) { } ~AOTCodeAddressTable(); void init_extrs(); - void init_early_stubs(); - void init_shared_blobs(); - void init_early_c1(); + void init_extrs2(); + void add_stub_entry(EntryId entry_id, address entry); + void add_external_addresses(GrowableArray
& addresses) NOT_CDS_RETURN; + void set_shared_stubs_complete(); + void set_c1_stubs_complete(); + void set_c2_stubs_complete(); + void set_stubgen_stubs_complete(); const char* add_C_string(const char* str); int id_for_C_string(address str); address address_for_C_string(int idx); @@ -163,6 +163,136 @@ class AOTCodeAddressTable : public CHeapObj { address address_for_id(int id); }; +// Auxiliary class used by AOTStubData to locate addresses owned by a +// stub in the _address_array. + +class StubAddrRange { +private: + // Index of the first address owned by a stub or -1 if none present + int _start_index; + // Total number of addresses owned by a stub, including in order: + // start address for stub code and first entry, (exclusive) end + // address for stub code, all secondary entry addresses, any + // auxiliary addresses + uint _naddr; + public: + StubAddrRange() : _start_index(-1), _naddr(0) {} + int start_index() { return _start_index; } + int count() { return _naddr; } + + void default_init() { + _start_index = -1; + _naddr = 0; + } + + void init_entry(int start_index, int naddr) { + _start_index = start_index; + _naddr = naddr; + } +}; + +// class used to save and restore details of stubs embedded in a +// multi-stub (StubGen) blob + +class AOTStubData : public StackObj { + friend class AOTCodeCache; + friend class AOTCodeReader; +private: + BlobId _blob_id; // must be a stubgen blob id + // whatever buffer blob was successfully loaded from the AOT cache + // following a call to load_code_blob or nullptr + CodeBlob *_cached_blob; + // Array of addresses owned by stubs. Each stub appends addresses to + // this array as a block, whether at the end of generation or at the + // end of restoration from the cache. The first two addresses in + // each block are the "start" and "end2 address of the stub. Any + // other visible addresses located within the range [start,end) + // follow, either extra entries, data addresses or SEGV-protected + // subrange start, end and handler addresses. In the special case + // that the SEGV handler address is the (external) common address + // handler the array will hold value nullptr. + GrowableArray
_address_array; + // count of how many stubs exist in the current blob (not all of + // which may actually be generated) + int _stub_cnt; + // array identifying range of entries in _address_array for each stub + // indexed by offset of stub in blob + StubAddrRange* _ranges; + // id of last looked up stub or NO_STUBID if lookup not attempted or failed + StubId _current; + // offset of _current in blob or -1 if lookup not attempted or failed + int _current_idx; + + // flags indicating whether the AOT code cache is open and, if so, + // whether we are loading or storing stubs.the first of those + // cases whether we have encountered any invalid stubs or failed to + // find a stub that was being generated + enum Flags { + OPEN = 1 << 0, // cache is open for use + USING = 1 << 1, // open and loading stubs + DUMPING = 1 << 2, // open and storing stubs + INVALID = 1 << 3, // found invalid stub when loading + }; + + uint32_t _flags; + + void set_invalid() { _flags |= INVALID; } + + StubAddrRange& get_range(int idx) const { return _ranges[idx]; } + GrowableArray
& address_array() { return _address_array; } + // accessor for entry/auxiliary addresses defaults to start entry +public: + AOTStubData(BlobId blob_id) NOT_CDS({}); + + ~AOTStubData() CDS_ONLY({FREE_C_HEAP_ARRAY(StubAddrRange, _ranges);}) NOT_CDS({}) + + bool is_open() CDS_ONLY({ return (_flags & OPEN) != 0; }) NOT_CDS_RETURN_(false); + bool is_using() CDS_ONLY({ return (_flags & USING) != 0; }) NOT_CDS_RETURN_(false); + bool is_dumping() CDS_ONLY({ return (_flags & DUMPING) != 0; }) NOT_CDS_RETURN_(false); + bool is_aot() CDS_ONLY({ return is_using() || is_dumping(); }) NOT_CDS_RETURN_(false); + bool is_invalid() CDS_ONLY({ return (_flags & INVALID) != 0; }) NOT_CDS_RETURN_(false); + + BlobId blob_id() { return _blob_id; } + StubId current_stub_id() { return _current; } + // + bool load_code_blob() NOT_CDS_RETURN_(true); + bool store_code_blob(CodeBlob& new_blob, CodeBuffer *code_buffer) NOT_CDS_RETURN_(true); + // determine whether a stub is available in the AOT cache + bool find_archive_data(StubId stub_id) NOT_CDS_RETURN_(false); + // retrieve stub entry data if it we are using archived stubs and + // the stub has been found in an AOT-restored blob or store stub + // entry data if we are saving archived stubs and the stub has just + // been successfully generated into the current blob. + // + // start and end identify the inclusive start and exclusive end + // address for stub code and must lie in the current blob's code + // range. Stubs presented via this interface must declare at least + // one entry and start is always taken to be the first entry. + // + // Optional arrays entries and extras present other addresses of + // interest all of which must either lie in the interval (start, + // end) or be nullptr (verified by load and store methods). + // + // entries lists secondary entries for the stub each of which must + // match a corresponding entry declaration for the stub (entry count + // verified by load and store methods). Null entry addresses are + // allowed when an architecture does not require a specific entry + // but may not vary from one run to the next. If the cache is in use + // at a store (for loading or saving code) then non-null entry + // addresses are entered into the AOT cache stub address table + // allowing references to them from other stubs or nmethods to be + // relocated. + // + // extras lists other non-entry stub addresses of interest such as + // memory protection ranges and associated handler addresses. These + // do do not need to be declared as entries and their number and + // meaning may vary according to the architecture. + void load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
* entries = nullptr, GrowableArray
* extras = nullptr) NOT_CDS_RETURN; + void store_archive_data(StubId stub_id, address start, address end, GrowableArray
* entries = nullptr, GrowableArray
* extras = nullptr) NOT_CDS_RETURN; + + const AOTStubData* as_const() { return (const AOTStubData*)this; } +}; + class AOTCodeCache : public CHeapObj { // Classes used to describe AOT code cache. @@ -204,16 +334,18 @@ class AOTCodeCache : public CHeapObj { uint _entries_offset; // offset of AOTCodeEntry array describing entries uint _adapters_count; uint _shared_blobs_count; + uint _stubgen_blobs_count; uint _C1_blobs_count; uint _C2_blobs_count; Config _config; public: void init(uint cache_size, - uint strings_count, uint strings_offset, - uint entries_count, uint entries_offset, - uint adapters_count, uint shared_blobs_count, - uint C1_blobs_count, uint C2_blobs_count) { + uint strings_count, uint strings_offset, + uint entries_count, uint entries_offset, + uint adapters_count, uint shared_blobs_count, + uint stubgen_blobs_count, uint C1_blobs_count, + uint C2_blobs_count) { _version = AOT_CODE_VERSION; _cache_size = cache_size; _strings_count = strings_count; @@ -222,6 +354,7 @@ class AOTCodeCache : public CHeapObj { _entries_offset = entries_offset; _adapters_count = adapters_count; _shared_blobs_count = shared_blobs_count; + _stubgen_blobs_count = stubgen_blobs_count; _C1_blobs_count = C1_blobs_count; _C2_blobs_count = C2_blobs_count; _config.record(); @@ -284,6 +417,7 @@ class AOTCodeCache : public CHeapObj { void clear_lookup_failed() { _lookup_failed = false; } bool lookup_failed() const { return _lookup_failed; } + void add_stub_entry(EntryId entry_id, address entry) NOT_CDS_RETURN; public: AOTCodeCache(bool is_dumping, bool is_using); ~AOTCodeCache(); @@ -300,9 +434,12 @@ class AOTCodeCache : public CHeapObj { void load_strings(); int store_strings(); - static void init_early_stubs_table() NOT_CDS_RETURN; - static void init_shared_blobs_table() NOT_CDS_RETURN; - static void init_early_c1_table() NOT_CDS_RETURN; + static void set_shared_stubs_complete() NOT_CDS_RETURN; + static void set_c1_stubs_complete() NOT_CDS_RETURN ; + static void set_c2_stubs_complete() NOT_CDS_RETURN; + static void set_stubgen_stubs_complete() NOT_CDS_RETURN; + + void add_stub_entries(StubId stub_id, address start, GrowableArray
*entries = nullptr, int offset = -1) NOT_CDS_RETURN; address address_for_C_string(int idx) const { return _table->address_for_C_string(idx); } address address_for_id(int id) const { return _table->address_for_id(id); } @@ -322,22 +459,41 @@ class AOTCodeCache : public CHeapObj { bool finish_write(); - bool write_relocations(CodeBlob& code_blob); + bool write_relocations(CodeBlob& code_blob, RelocIterator& iter); bool write_oop_map_set(CodeBlob& cb); + bool write_stub_data(CodeBlob& blob, AOTStubData *stub_data); #ifndef PRODUCT bool write_asm_remarks(CodeBlob& cb); bool write_dbg_strings(CodeBlob& cb); #endif // PRODUCT +private: + // internal private API to save and restore blobs + static bool store_code_blob(CodeBlob& blob, + AOTCodeEntry::Kind entry_kind, + uint id, + const char* name, + AOTStubData* stub_data, + CodeBuffer* code_buffer) NOT_CDS_RETURN_(false); + + static CodeBlob* load_code_blob(AOTCodeEntry::Kind kind, + uint id, + const char* name, + AOTStubData* stub_data) NOT_CDS_RETURN_(nullptr); + +public: // save and restore API for non-enumerable code blobs static bool store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, - uint id, const char* name) NOT_CDS_RETURN_(false); + uint id, + const char* name) NOT_CDS_RETURN_(false); static CodeBlob* load_code_blob(AOTCodeEntry::Kind kind, uint id, const char* name) NOT_CDS_RETURN_(nullptr); // save and restore API for enumerable code blobs + + // API for single-stub blobs static bool store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id) NOT_CDS_RETURN_(false); @@ -345,6 +501,22 @@ class AOTCodeCache : public CHeapObj { static CodeBlob* load_code_blob(AOTCodeEntry::Kind kind, BlobId id) NOT_CDS_RETURN_(nullptr); + // API for multi-stub blobs -- for use by class StubGenerator. + + static bool store_code_blob(CodeBlob& blob, + AOTCodeEntry::Kind kind, + BlobId id, + AOTStubData* stub_data, + CodeBuffer *code_buffer) NOT_CDS_RETURN_(false); + + static CodeBlob* load_code_blob(AOTCodeEntry::Kind kind, + BlobId id, + AOTStubData* stub_data) NOT_CDS_RETURN_(nullptr); + + static void publish_external_addresses(GrowableArray
& addresses) NOT_CDS_RETURN; + // publish all entries for a code blob in code cache address table + static void publish_stub_addresses(CodeBlob &code_blob, BlobId id, AOTStubData *stub_data) NOT_CDS_RETURN; + static uint store_entries_cnt() { if (is_on_for_dump()) { return cache()->_store_entries_cnt; @@ -369,6 +541,7 @@ class AOTCodeCache : public CHeapObj { static AOTCodeCache* cache() { assert(_passed_init2, "Too early to ask"); return _cache; } static void initialize() NOT_CDS_RETURN; static void init2() NOT_CDS_RETURN; + static void init3() NOT_CDS_RETURN; static void close() NOT_CDS_RETURN; static bool is_on() CDS_ONLY({ return cache() != nullptr && !_cache->closing(); }) NOT_CDS_RETURN_(false); static bool is_on_for_use() CDS_ONLY({ return is_on() && _cache->for_use(); }) NOT_CDS_RETURN_(false); @@ -389,7 +562,7 @@ class AOTCodeCache : public CHeapObj { // Concurent AOT code reader class AOTCodeReader { private: - const AOTCodeCache* _cache; + AOTCodeCache* _cache; const AOTCodeEntry* _entry; const char* _load_buffer; // Loaded cached code buffer uint _read_position; // Position in _load_buffer @@ -406,11 +579,14 @@ class AOTCodeReader { public: AOTCodeReader(AOTCodeCache* cache, AOTCodeEntry* entry); - CodeBlob* compile_code_blob(const char* name); + CodeBlob* compile_code_blob(const char* name, AOTCodeEntry::Kind entry_kind, int id, AOTStubData* stub_data = nullptr); ImmutableOopMapSet* read_oop_map_set(); + void read_stub_data(CodeBlob* code_blob, AOTStubData *stub_data); + void publish_stub_addresses(CodeBlob &code_blob, BlobId id, AOTStubData *stub_data); + - void fix_relocations(CodeBlob* code_blob); + void fix_relocations(CodeBlob* code_blob, RelocIterator& iter); #ifndef PRODUCT void read_asm_remarks(AsmRemarks& asm_remarks); void read_dbg_strings(DbgStrings& dbg_strings); diff --git a/src/hotspot/share/prims/downcallLinker.hpp b/src/hotspot/share/prims/downcallLinker.hpp index 01ee5c5677666..b27b18aff2588 100644 --- a/src/hotspot/share/prims/downcallLinker.hpp +++ b/src/hotspot/share/prims/downcallLinker.hpp @@ -71,7 +71,7 @@ class DowncallLinker: AllStatic { bool needs_return_buffer, int captured_state_mask, bool needs_transition) - : StubCodeGenerator(buffer, PrintMethodHandleStubs), + : StubCodeGenerator(buffer, PrintMethodHandleStubs), _signature(signature), _num_args(num_args), _ret_bt(ret_bt), diff --git a/src/hotspot/share/runtime/init.cpp b/src/hotspot/share/runtime/init.cpp index adc49f84358b3..463460fe3d57a 100644 --- a/src/hotspot/share/runtime/init.cpp +++ b/src/hotspot/share/runtime/init.cpp @@ -70,6 +70,8 @@ void VM_Version_init(); void icache_init2(); void initialize_stub_info(); // must precede all blob/stub generation void preuniverse_stubs_init(); + +void stubs_AOTAddressTable_init(); void initial_stubs_init(); jint universe_init(); // depends on codeCache_init and preuniverse_stubs_init @@ -149,13 +151,17 @@ jint init_globals() { AOTCodeCache::init2(); // depends on universe_init, must be before initial_stubs_init AsyncLogWriter::initialize(); + stubs_AOTAddressTable_init(); // publish external addresses used by stubs + // depends on AOTCodeCache::init2 initial_stubs_init(); // stubgen initial stub routines // stack overflow exception blob is referenced by the interpreter - AOTCodeCache::init_early_stubs_table(); // need this after stubgen initial stubs and before shared runtime initial stubs SharedRuntime::generate_initial_stubs(); gc_barrier_stubs_init(); // depends on universe_init, must be before interpreter_init continuations_init(); // must precede continuation stub generation - continuation_stubs_init(); // depends on continuations_init + AOTCodeCache::init3(); // depends on stubs_AOTAddressTable_init + // and continuations_init and must + // precede continuation stub generation + continuation_stubs_init(); // depends on continuations_init and AOTCodeCache::init3 #if INCLUDE_JFR SharedRuntime::generate_jfr_stubs(); #endif @@ -164,7 +170,6 @@ jint init_globals() { InterfaceSupport_init(); VMRegImpl::set_regName(); // need this before generate_stubs (for printing oop maps). SharedRuntime::generate_stubs(); - AOTCodeCache::init_shared_blobs_table(); // need this after generate_stubs SharedRuntime::init_adapter_library(); // do this after AOTCodeCache::init_shared_blobs_table return JNI_OK; } diff --git a/src/hotspot/share/runtime/stubCodeGenerator.cpp b/src/hotspot/share/runtime/stubCodeGenerator.cpp index 43250c004ca3d..4859a110ca2ce 100644 --- a/src/hotspot/share/runtime/stubCodeGenerator.cpp +++ b/src/hotspot/share/runtime/stubCodeGenerator.cpp @@ -23,6 +23,7 @@ */ #include "asm/macroAssembler.inline.hpp" +#include "code/aotCodeCache.hpp" #include "code/codeCache.hpp" #include "compiler/disassembler.hpp" #include "oops/oop.inline.hpp" @@ -69,14 +70,16 @@ void StubCodeDesc::print() const { print_on(tty); } StubCodeGenerator::StubCodeGenerator(CodeBuffer* code, bool print_code) { _masm = new MacroAssembler(code); _blob_id = BlobId::NO_BLOBID; + _stub_data = nullptr; _print_code = PrintStubCode || print_code; } -StubCodeGenerator::StubCodeGenerator(CodeBuffer* code, BlobId blob_id, bool print_code) { +StubCodeGenerator::StubCodeGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data, bool print_code) { assert(StubInfo::is_stubgen(blob_id), "not a stubgen blob %s", StubInfo::name(blob_id)); _masm = new MacroAssembler(code); _blob_id = blob_id; + _stub_data = stub_data; _print_code = PrintStubCode || print_code; } @@ -91,11 +94,59 @@ StubCodeGenerator::~StubCodeGenerator() { #endif } +void StubCodeGenerator::setup_code_desc(const char* name, address start, address end, bool loaded_from_cache) { + StubCodeDesc* cdesc = new StubCodeDesc("StubRoutines", name, start, end); + cdesc->set_disp(uint(start - _masm->code_section()->outer()->insts_begin())); + if (loaded_from_cache) { + cdesc->set_loaded_from_cache(); + } + print_stub_code_desc(cdesc); + // copied from ~StubCodeMark() + Forte::register_stub(cdesc->name(), cdesc->begin(), cdesc->end()); + if (JvmtiExport::should_post_dynamic_code_generated()) { + JvmtiExport::post_dynamic_code_generated(cdesc->name(), cdesc->begin(), cdesc->end()); + } +} + +// Helper used to restore ranges and handler addresses restored from +// AOT cache. Expects entries to contain 3 * count addresses beginning +// at offset begin which identify start of range, end of range and +// address of handler pc. end of range may be nullptr in which case it +// defaults to stub_end. hanlder pc may be nullptr in which case it +// defaults to default_handler. + +void StubCodeGenerator::register_unsafe_access_handlers(GrowableArray
&entries, int begin, int count, address stub_end) { + for (int i = 0; i < count; i++) { + int offset = begin + 3 * i; + address start = entries.at(offset); + address end = entries.at(offset + 1); + if (end == nullptr) { + end = stub_end; + } + address handler = entries.at(offset + 2); + if (handler == nullptr) { + handler = UnsafeMemoryAccess::common_exit_stub_pc(); + } + UnsafeMemoryAccess::add_to_table(start, end, handler); + } +} + +// Helper used to retrieve ranges and handler addresses registered +// during generation of the stub which spans [start, end) in order to +// allow them to be saved to an AOT cache. +void StubCodeGenerator::retrieve_unsafe_access_handlers(address start, address end, GrowableArray
&entries) { + UnsafeMemoryAccess::collect_entries(start, end, entries); +} + void StubCodeGenerator::stub_prolog(StubCodeDesc* cdesc) { // default implementation - do nothing } void StubCodeGenerator::stub_epilog(StubCodeDesc* cdesc) { + print_stub_code_desc(cdesc); +} + +void StubCodeGenerator::print_stub_code_desc(StubCodeDesc* cdesc) { LogTarget(Debug, stubs) lt; if (lt.is_enabled()) { LogStream ls(lt); @@ -119,6 +170,53 @@ void StubCodeGenerator::stub_epilog(StubCodeDesc* cdesc) { } } +bool StubCodeGenerator::find_archive_data(StubId stub_id) { + // punt to stub data if it exists and is not for dumping + if (_stub_data == nullptr || _stub_data->is_dumping()) { + return false; + } + return _stub_data->find_archive_data(stub_id); +} + +void StubCodeGenerator::load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
*entries, GrowableArray
* extras) { + assert(_stub_data != nullptr && _stub_data->current_stub_id() == stub_id, "no current archive data for %s", StubInfo::name(stub_id)); + + // punt to stub data + _stub_data->load_archive_data(stub_id, start, end, entries, extras); + + setup_code_desc(StubInfo::name(stub_id), start, end, true); +} + +void StubCodeGenerator::store_archive_data(StubId stub_id, address start, address end, GrowableArray
* entries, GrowableArray
* extras) { + // punt to stub data if we have any + if (_stub_data != nullptr) { + _stub_data->store_archive_data(stub_id, start, end, entries, extras); + } +} + +void StubCodeGenerator::print_statistics_on(outputStream* st) { + st->print_cr("StubRoutines Stubs:"); + st->print_cr(" Initial stubs: %d", StubInfo::stub_count(BlobId::stubgen_initial_id)); + st->print_cr(" Continuation stubs: %d", StubInfo::stub_count(BlobId::stubgen_continuation_id)); + st->print_cr(" Compiler stubs: %d", StubInfo::stub_count(BlobId::stubgen_compiler_id)); + st->print_cr(" Final stubs: %d", StubInfo::stub_count(BlobId::stubgen_final_id)); + + int emitted = 0; + int loaded_from_cache = 0; + + StubCodeDesc* scd = StubCodeDesc::first(); + while (scd != nullptr) { + if (!strcmp(scd->group(), "StubRoutines")) { + emitted += 1; + if (scd->loaded_from_cache()) { + loaded_from_cache += 1; + } + } + scd = StubCodeDesc::next(scd); + } + st->print_cr("Total stubroutines stubs emitted: %d (generated=%d, loaded from cache=%d)", emitted, emitted - loaded_from_cache, loaded_from_cache); +} + #ifdef ASSERT void StubCodeGenerator::verify_stub(StubId stub_id) { assert(StubRoutines::stub_to_blob(stub_id) == blob_id(), "wrong blob %s for generation of stub %s", StubRoutines::get_blob_name(blob_id()), StubRoutines::get_stub_name(stub_id)); diff --git a/src/hotspot/share/runtime/stubCodeGenerator.hpp b/src/hotspot/share/runtime/stubCodeGenerator.hpp index 7d8944c85eada..18cf461f27e35 100644 --- a/src/hotspot/share/runtime/stubCodeGenerator.hpp +++ b/src/hotspot/share/runtime/stubCodeGenerator.hpp @@ -26,6 +26,7 @@ #define SHARE_RUNTIME_STUBCODEGENERATOR_HPP #include "asm/assembler.hpp" +#include "code/aotCodeCache.hpp" #include "memory/allocation.hpp" #include "runtime/stubInfo.hpp" @@ -48,6 +49,7 @@ class StubCodeDesc: public CHeapObj { address _begin; // points to the first byte of the stub code (included) address _end; // points to the first byte after the stub code (excluded) uint _disp; // Displacement relative base address in buffer. + bool _loaded_from_cache; friend class StubCodeMark; friend class StubCodeGenerator; @@ -65,6 +67,8 @@ class StubCodeDesc: public CHeapObj { void set_disp(uint disp) { _disp = disp; } + void set_loaded_from_cache() { _loaded_from_cache = true; } + public: static StubCodeDesc* first() { return _list; } static StubCodeDesc* next(StubCodeDesc* desc) { return desc->_next; } @@ -81,6 +85,7 @@ class StubCodeDesc: public CHeapObj { _end = end; _disp = 0; _list = this; + _loaded_from_cache = false; }; static void freeze(); @@ -93,12 +98,11 @@ class StubCodeDesc: public CHeapObj { uint disp() const { return _disp; } int size_in_bytes() const { return pointer_delta_as_int(_end, _begin); } bool contains(address pc) const { return _begin <= pc && pc < _end; } + bool loaded_from_cache() const { return _loaded_from_cache; } void print_on(outputStream* st) const; void print() const; }; -// forward declare blob and stub id enums - // The base class for all stub-generating code generators. // Provides utility functions. @@ -108,10 +112,16 @@ class StubCodeGenerator: public StackObj { BlobId _blob_id; protected: MacroAssembler* _masm; + AOTStubData* _stub_data; + + void setup_code_desc(const char* name, address start, address end, bool loaded_from_cache); + // unsafe handler management + void register_unsafe_access_handlers(GrowableArray
&entries, int begin, int count, address stub_end); + void retrieve_unsafe_access_handlers(address start, address end, GrowableArray
&entries); public: StubCodeGenerator(CodeBuffer* code, bool print_code = false); - StubCodeGenerator(CodeBuffer* code, BlobId blob_id, bool print_code = false); + StubCodeGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data = nullptr, bool print_code = false); ~StubCodeGenerator(); MacroAssembler* assembler() const { return _masm; } @@ -120,9 +130,16 @@ class StubCodeGenerator: public StackObj { virtual void stub_prolog(StubCodeDesc* cdesc); // called by StubCodeMark constructor virtual void stub_epilog(StubCodeDesc* cdesc); // called by StubCodeMark destructor + void print_stub_code_desc(StubCodeDesc* cdesc); + + static void print_statistics_on(outputStream* st); + bool find_archive_data(StubId stub_id); + void load_archive_data(StubId stub_id, address& start, address& end, GrowableArray
*entries = nullptr, GrowableArray
* extras = nullptr); + void store_archive_data(StubId stub_id, address start, address end, GrowableArray
*entries = nullptr, GrowableArray
* extras = nullptr); #ifdef ASSERT void verify_stub(StubId stub_id); #endif + }; // Stack-allocated helper class used to associate a stub code with a name. diff --git a/src/hotspot/share/runtime/stubDeclarations.hpp b/src/hotspot/share/runtime/stubDeclarations.hpp index b051c6d0e184e..15735797d7c68 100644 --- a/src/hotspot/share/runtime/stubDeclarations.hpp +++ b/src/hotspot/share/runtime/stubDeclarations.hpp @@ -975,9 +975,15 @@ do_entry_init(final, arrayof_jlong_arraycopy, \ arrayof_jlong_arraycopy, arrayof_jlong_arraycopy, \ StubRoutines::arrayof_jlong_copy) \ + do_entry(final, arrayof_jlong_arraycopy, \ + arrayof_jlong_arraycopy_nopush, \ + arrayof_jlong_arraycopy_nopush) \ do_stub(final, arrayof_oop_arraycopy) \ do_entry_init(final, arrayof_oop_arraycopy, arrayof_oop_arraycopy, \ arrayof_oop_arraycopy, StubRoutines::arrayof_oop_copy) \ + do_entry(final, arrayof_oop_arraycopy, \ + arrayof_oop_arraycopy_nopush, \ + arrayof_oop_arraycopy_nopush) \ do_stub(final, arrayof_oop_arraycopy_uninit) \ do_entry_init(final, arrayof_oop_arraycopy_uninit, \ arrayof_oop_arraycopy_uninit, \ diff --git a/src/hotspot/share/runtime/stubInfo.cpp b/src/hotspot/share/runtime/stubInfo.cpp index bca6ff344ea85..6c7769dd66d8d 100644 --- a/src/hotspot/share/runtime/stubInfo.cpp +++ b/src/hotspot/share/runtime/stubInfo.cpp @@ -1096,6 +1096,15 @@ int StubInfo::stubgen_offset(StubId id) { return local_offset(StubGroup::STUBGEN, id); } +int StubInfo::stubgen_offset_in_blob(BlobId blob_id, StubId id) { + assert(blob(id) == blob_id, "sanity!"); + StubGroup group = StubGroup::STUBGEN; + assert(stubgroup(blob_id) == group, "sanity"); + StubId base_id = stub_base(blob_id); + assert(base_id != StubId::NO_STUBID, "sanity"); + return local_offset(group, id) - local_offset(group, base_id); +} + // initialization function called to populate blob. stub and entry // tables. this must be called before any stubs are generated void initialize_stub_info() { diff --git a/src/hotspot/share/runtime/stubInfo.hpp b/src/hotspot/share/runtime/stubInfo.hpp index eaea07f7e6c02..323d006e850c4 100644 --- a/src/hotspot/share/runtime/stubInfo.hpp +++ b/src/hotspot/share/runtime/stubInfo.hpp @@ -678,6 +678,11 @@ class StubInfo: AllStatic { static int c1_offset(StubId id); static int c2_offset(StubId id); static int stubgen_offset(StubId id); + + // Convert a stub id to a unique, zero-based offset in the range of + // stub ids for a given blob in the stubgen stub group. + + static int stubgen_offset_in_blob(BlobId blob_id, StubId id); }; diff --git a/src/hotspot/share/runtime/stubRoutines.cpp b/src/hotspot/share/runtime/stubRoutines.cpp index 5246613738e46..84d469af5c513 100644 --- a/src/hotspot/share/runtime/stubRoutines.cpp +++ b/src/hotspot/share/runtime/stubRoutines.cpp @@ -102,7 +102,10 @@ BlobId StubRoutines::stub_to_blob(StubId id) { // Initialization -extern void StubGenerator_generate(CodeBuffer* code, BlobId blob_id); // only interface to generators +extern void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data); // only interface to generators +#if (defined(X86) && defined(_LP64)) || defined(AARCH64) +extern void StubGenerator_AOTAddressTable_init(); +#endif void UnsafeMemoryAccess::create_table(int max_size) { UnsafeMemoryAccess::_table = new UnsafeMemoryAccess[max_size]; @@ -169,6 +172,33 @@ static BufferBlob* initialize_stubs(BlobId blob_id, assert(StubInfo::is_stubgen(blob_id), "not a stubgen blob %s", StubInfo::name(blob_id)); ResourceMark rm; TraceTime timer(timer_msg, TRACETIME_LOG(Info, startuptime)); + // If we are loading stubs we need to check if we can retrieve a + // blob and/or an associated archived stub descriptor from the + // AOTCodeCache. If we are storing stubs we need to create a blob + // but we still need a stub data descriptor to fill in during + // generation. + AOTStubData stub_data(blob_id); + AOTStubData* stub_data_p = nullptr; + LogTarget(Info, stubs) lt; + + if (code_size > 0 && stub_data.is_using()) { + // AOTCodeEntry tracks and logs status of any cached blob + bool loaded = stub_data.load_code_blob(); + if (loaded) { + if (lt.is_enabled()) { + LogStream ls(lt); + ls.print_cr("Found blob %s in AOT cache", StubInfo::name(blob_id)); + } + stub_data_p = &stub_data; + } + } else if (stub_data.is_dumping()) { + stub_data_p = &stub_data; + } + + // Even if we managed to load a blob from the AOT cache we still + // need to allocate a code blob and associated buffer. The AOT blob + // may not include all the stubs we need for this runtime. + // Add extra space for large CodeEntryAlignment int size = code_size + CodeEntryAlignment * max_aligned_stubs; BufferBlob* stubs_code = BufferBlob::create(buffer_name, size); @@ -178,6 +208,10 @@ static BufferBlob* initialize_stubs(BlobId blob_id, // In that case we can tolerate an allocation failure because the // compiler will have been shut down and we have no need of the // blob. + // TODO: Ideally we would still like to try to use any AOT cached + // blob here but we don't have a fallback if we find that it is + // missing stubs we need so for now we exit. This should only + // happen in cases where we have a very small code cache. if (Thread::current()->is_Compiler_thread()) { assert(blob_id == BlobId::stubgen_compiler_id, "sanity"); assert(DelayCompilerStubsGeneration, "sanity"); @@ -187,10 +221,12 @@ static BufferBlob* initialize_stubs(BlobId blob_id, vm_exit_out_of_memory(code_size, OOM_MALLOC_ERROR, "CodeCache: no room for %s", buffer_name); } CodeBuffer buffer(stubs_code); - StubGenerator_generate(&buffer, blob_id); + short buffer_locs[20]; + buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs, + sizeof(buffer_locs)/sizeof(relocInfo)); + StubGenerator_generate(&buffer, blob_id, stub_data_p); if (code_size == 0) { assert(buffer.insts_size() == 0, "should not write into buffer when bob size declared as 0"); - LogTarget(Info, stubs) lt; if (lt.is_enabled()) { LogStream ls(lt); ls.print_cr("%s\t not generated", buffer_name); @@ -203,7 +239,35 @@ static BufferBlob* initialize_stubs(BlobId blob_id, "increase %s, code_size: %d, used: %d, free: %d", assert_msg, code_size, buffer.total_content_size(), buffer.insts_remaining()); - LogTarget(Info, stubs) lt; + if (stub_data.is_using()) { + // we generated some new entries so republish all entries TODO - + // ensure we publish collect and publish the preuniverse stubs but + // don't try to save them + AOTCodeCache::publish_stub_addresses(*stubs_code, blob_id, &stub_data); + if (lt.is_enabled()) { + LogStream ls(lt); + ls.print_cr("Republished entries for blob '%s'", buffer_name); + } + } else if (stub_data.is_dumping()) { + // save the blob and publihs the entry addresses + if (stub_data.store_code_blob(*stubs_code, &buffer)) { + if (lt.is_enabled()) { + LogStream ls(lt); + ls.print_cr("Stored blob '%s' to Startup Code Cache", buffer_name); + } + } else { + if (lt.is_enabled()) { + LogStream ls(lt); + ls.print_cr("Failed to store blob '%s' to Startup Code Cache", buffer_name); + } + } + } + + // close off recording of any further stubgen generation + if (blob_id == BlobId::stubgen_final_id) { + AOTCodeCache::set_stubgen_stubs_complete(); + } + if (lt.is_enabled()) { LogStream ls(lt); ls.print_cr("%s\t [" INTPTR_FORMAT ", " INTPTR_FORMAT "] used: %d, free: %d", @@ -214,6 +278,8 @@ static BufferBlob* initialize_stubs(BlobId blob_id, return stubs_code; } +// per blob initializer methods StubRoutines::initialize_xxx_stubs() + #define DEFINE_BLOB_INIT_METHOD(blob_name) \ void StubRoutines::initialize_ ## blob_name ## _stubs() { \ if (STUBGEN_BLOB_FIELD_NAME(blob_name) == nullptr) { \ @@ -234,6 +300,7 @@ STUBGEN_BLOBS_DO(DEFINE_BLOB_INIT_METHOD) #undef DEFINE_BLOB_INIT_METHOD +// external driver API functions for per blob init: xxx_stubs_init() #define DEFINE_BLOB_INIT_FUNCTION(blob_name) \ void blob_name ## _stubs_init() { \ @@ -244,11 +311,24 @@ STUBGEN_BLOBS_DO(DEFINE_BLOB_INIT_FUNCTION) #undef DEFINE_BLOB_INIT_FUNCTION + +// Non-generated init method + +void StubRoutines::init_AOTAddressTable() { +#if (defined(X86) && defined(_LP64)) || defined(AARCH64) + StubGenerator_AOTAddressTable_init(); +#endif +} + +// non-generated external API init driver function + +void stubs_AOTAddressTable_init() { StubRoutines::init_AOTAddressTable(); } + /* - * we generate the underlying driver method but this wrapper is needed - * to perform special handling depending on where the compiler init - * gets called from. it ought to be possible to remove this at some - * point and have a determinate ordered init. + * we generate the underlying driver function compiler_stubs_init() + * but this wrapper is needed to perform special handling depending on + * where the compiler init gets called from. it ought to be possible + * to remove this at some point and have a determinate ordered init. */ void compiler_stubs_init(bool in_compiler_thread) { diff --git a/src/hotspot/share/runtime/stubRoutines.hpp b/src/hotspot/share/runtime/stubRoutines.hpp index 97e3e46b87063..b2384c33a998d 100644 --- a/src/hotspot/share/runtime/stubRoutines.hpp +++ b/src/hotspot/share/runtime/stubRoutines.hpp @@ -130,6 +130,7 @@ class UnsafeMemoryAccess : public CHeapObj { static UnsafeMemoryAccess* add_to_table(address start_pc, address end_pc, address error_exit_pc) { guarantee(_table_length < _table_max_length, "Incorrect UnsafeMemoryAccess::_table_max_length"); UnsafeMemoryAccess* entry = &_table[_table_length]; + assert(start_pc != nullptr, "invalid start address"); entry->set_start_pc(start_pc); entry->set_end_pc(end_pc); entry->set_error_exit_pc(error_exit_pc); @@ -283,6 +284,9 @@ class StubRoutines: AllStatic { static BlobId stub_to_blob(StubId id); #endif + // Initalization + static void init_AOTAddressTable(); + // Debugging static jint verify_oop_count() { return _verify_oop_count; } static jint* verify_oop_count_addr() { return &_verify_oop_count; } diff --git a/test/hotspot/jtreg/runtime/cds/appcds/aotCode/AOTCodeFlags.java b/test/hotspot/jtreg/runtime/cds/appcds/aotCode/AOTCodeFlags.java index 761c2c5382d61..4219cbdf6be88 100644 --- a/test/hotspot/jtreg/runtime/cds/appcds/aotCode/AOTCodeFlags.java +++ b/test/hotspot/jtreg/runtime/cds/appcds/aotCode/AOTCodeFlags.java @@ -51,7 +51,7 @@ public class AOTCodeFlags { public static void main(String... args) throws Exception { Tester t = new Tester(); // Run only 2 modes (0 - no AOT code, 1 - AOT adapters) until JDK-8357398 is fixed - for (int mode = 0; mode < 2; mode++) { + for (int mode = 0; mode < 4; mode++) { t.setTestMode(mode); t.run(new String[] {"AOT", "--two-step-training"}); }