8371864: GaloisCounterMode.implGCMCrypt0 AVX512/AVX2 intrinsics stubs cause AES-GCM encryption failure for certain payload sizes

Co-authored-by: Thomas Holenstein <[email protected]>
Co-authored-by: Lukas Zobernig <[email protected]>
Reviewed-by: shade, sviswanathan

Author: 3 people

src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp

@@ -3524,10 +3524,10 @@ void StubGenerator::aesgcm_avx512(Register in, Register len, Register ct, Regist
                                     false, true, false, false, false, ghashin_offset, aesout_offset, HashKey_32);
 
   ghash16_avx512(false, true, false, false, true, in, pos, avx512_subkeyHtbl, AAD_HASHx, SHUF_MASK, stack_offset, 16 * 16, 0, HashKey_16);
+  __ addl(pos, 16 * 16);
 
   __ bind(MESG_BELOW_32_BLKS);
   __ subl(len, 16 * 16);
-  __ addl(pos, 16 * 16);
   gcm_enc_dec_last_avx512(len, in, pos, AAD_HASHx, SHUF_MASK, avx512_subkeyHtbl, ghashin_offset, HashKey_16, true, true);
 
   __ bind(GHASH_DONE);
@@ -4016,13 +4016,15 @@ void StubGenerator::aesgcm_avx2(Register in, Register len, Register ct, Register
   const Register rounds = r10;
   const XMMRegister ctr_blockx = xmm9;
   const XMMRegister aad_hashx = xmm8;
-  Label encrypt_done, encrypt_by_8_new, encrypt_by_8;
+  Label encrypt_done, encrypt_by_8_new, encrypt_by_8, exit;
 
   //This routine should be called only for message sizes of 128 bytes or more.
   //Macro flow:
   //process 8 16 byte blocks in initial_num_blocks.
   //process 8 16 byte blocks at a time until all are done 'encrypt_by_8_new  followed by ghash_last_8'
   __ xorl(pos, pos);
+  __ cmpl(len, 128);
+  __ jcc(Assembler::less, exit);
 
   //Generate 8 constants for htbl
   generateHtbl_8_block_avx2(subkeyHtbl);
@@ -4090,6 +4092,7 @@ void StubGenerator::aesgcm_avx2(Register in, Register len, Register ct, Register
   __ vpxor(xmm0, xmm0, xmm0, Assembler::AVX_128bit);
   __ vpxor(xmm13, xmm13, xmm13, Assembler::AVX_128bit);
 
+  __ bind(exit);
  }
 
 #undef __

test/jdk/com/sun/crypto/provider/Cipher/AES/TestGCMSplitBound.java

@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2025, Google LLC. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8371864
+ * @run main/othervm/timeout=600 TestGCMSplitBound
+ * @requires (os.simpleArch == "x64" & (vm.cpu.features ~= ".*avx2.*" |
+ *                                      vm.cpu.features ~= ".*avx512.*"))
+ * @summary Test GaloisCounterMode.implGCMCrypt0 AVX512/AVX2 intrinsics.
+ */
+
+import java.security.SecureRandom;
+import java.security.spec.AlgorithmParameterSpec;
+import java.time.Duration;
+import java.util.Arrays;
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.GCMParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
+
+public class TestGCMSplitBound {
+
+    static final SecureRandom SECURE_RANDOM = newDefaultSecureRandom();
+
+    private static SecureRandom newDefaultSecureRandom() {
+        SecureRandom retval = new SecureRandom();
+        retval.nextLong(); // force seeding
+        return retval;
+    }
+
+    private static byte[] randBytes(int size) {
+        byte[] rand = new byte[size];
+        SECURE_RANDOM.nextBytes(rand);
+        return rand;
+    }
+
+    private static final int IV_SIZE_IN_BYTES = 12;
+    private static final int TAG_SIZE_IN_BYTES = 16;
+
+    private Cipher getCipher(final byte[] key, final byte[] aad,
+                             final byte[] nonce, int mode)
+        throws Exception {
+        SecretKey keySpec = new SecretKeySpec(key, "AES");
+        AlgorithmParameterSpec params =
+            new GCMParameterSpec(8 * TAG_SIZE_IN_BYTES, nonce, 0, nonce.length);
+        Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
+        cipher.init(mode, keySpec, params);
+        if (aad != null && aad.length != 0) {
+            cipher.updateAAD(aad);
+        }
+        return cipher;
+    }
+
+    private byte[] gcmEncrypt(final byte[] key, final byte[] plaintext,
+                              final byte[] aad)
+        throws Exception {
+        byte[] nonce = randBytes(IV_SIZE_IN_BYTES);
+        Cipher cipher = getCipher(key, aad, nonce, Cipher.ENCRYPT_MODE);
+        int outputSize = cipher.getOutputSize(plaintext.length);
+        int len = IV_SIZE_IN_BYTES + outputSize;
+        byte[] output = new byte[len];
+        System.arraycopy(nonce, 0, output, 0, IV_SIZE_IN_BYTES);
+        cipher.doFinal(plaintext, 0, plaintext.length, output,
+                       IV_SIZE_IN_BYTES);
+        return output;
+    }
+
+    private byte[] gcmDecrypt(final byte[] key, final byte[] ciphertext,
+                              final byte[] aad)
+        throws Exception {
+        byte[] nonce = new byte[IV_SIZE_IN_BYTES];
+        System.arraycopy(ciphertext, 0, nonce, 0, IV_SIZE_IN_BYTES);
+        Cipher cipher = getCipher(key, aad, nonce, Cipher.DECRYPT_MODE);
+        return cipher.doFinal(ciphertext, IV_SIZE_IN_BYTES,
+                              ciphertext.length - IV_SIZE_IN_BYTES);
+    }
+
+    // x86-64 parallel intrinsic data size
+    private static final int PARALLEL_LEN = 512;
+    // max data size for x86-64 intrinsic
+    private static final int SPLIT_LEN = 1048576; // 1MB
+
+    private void encryptAndDecrypt(byte[] key, byte[] aad, byte[] message,
+                                   int messageSize)
+        throws Exception {
+        byte[] ciphertext = gcmEncrypt(key, message, aad);
+        byte[] decrypted = gcmDecrypt(key, ciphertext, aad);
+        if (ciphertext == null) {
+            throw new RuntimeException("ciphertext is null");
+        }
+        if (Arrays.compare(decrypted, 0, messageSize,
+                           message, 0, messageSize) != 0) {
+            throw new RuntimeException(
+                 "Decrypted message is different from the original message");
+        }
+    }
+
+    private void run() throws Exception {
+        byte[] aad = randBytes(20);
+        byte[] key = randBytes(16);
+        // Force JIT.
+        for (int i = 0; i < 100000; i++) {
+            byte[] message = randBytes(PARALLEL_LEN);
+            encryptAndDecrypt(key, aad, message, PARALLEL_LEN);
+        }
+        for (int messageSize = SPLIT_LEN - 300; messageSize <= SPLIT_LEN + 300;
+                                                messageSize++) {
+            byte[] message = randBytes(messageSize);
+            try {
+                encryptAndDecrypt(key, aad, message, messageSize);
+            } catch (Exception e) {
+                throw new RuntimeException("Failed for messageSize "
+                        + Integer.toHexString(messageSize), e);
+            }
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        TestGCMSplitBound test = new TestGCMSplitBound();
+        for (int i = 0; i < 3; i++) {
+            test.run();
+        }
+    }
+}