[GR-59869] Add SIMD code generation and Vector API support

PullRequest: graal/20854

Author: gergo-

compiler/CHANGELOG.md

@@ -8,6 +8,13 @@ This changelog summarizes newly introduced optimizations and other compiler rela
   which has no dependency on Native Image internals. This
   is required for Galahad CE where libgraal must be buildable from the Graal compiler sources in the OpenJDK
   while using Native Image as an external tool.
+* (GR-59869) Implemented initial optimization of Java Vector API (JEP 338) operations.
+  Load, store, basic arithmetic, reduce, compare, and blend operations are transformed to efficient machine instructions where possible.
+  Coverage of more operations is planned for the future.
+  This optimization is experimental.
+  It is enabled by default and can be disabled by setting the `OptimizeVectorAPI` option to `false`.
+  Vector API operations are supported both on JIT and when building native images.
+  Native image builds must use the `--add-modules jdk.incubator.vector` and `-H:+VectorAPISupport` options to enable optimization.
 
 ## GraalVM for JDK 24 (Internal Version 24.2.0)
 * (GR-57209): The default number of JVMCI threads is now the same as the number of C2 threads (`-XX:JVMCINativeLibraryThreadFraction=0.66`).

compiler/mx.compiler/suite.py

@@ -262,6 +262,7 @@
         "java.instrument",
         "java.management",
         "jdk.jfr",
+        "jdk.incubator.vector",
       ],
       "requiresConcealed" : {
         "java.base" : [
@@ -297,6 +298,8 @@
       "checkstyle" : "jdk.graal.compiler",
       "javaCompliance" : "21+",
       "jacoco" : "exclude",
+      # warning: [incubating] using incubating module(s): jdk.incubator.vector
+      "javac.lint.overrides" : "-incubating",
       "graalCompilerSourceEdition": "ignore",
     },
 

compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/VectorAPISmokeTest.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package jdk.graal.compiler.core.test;
+
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+
+import jdk.graal.compiler.core.common.type.IntegerStamp;
+import jdk.graal.compiler.core.common.type.StampFactory;
+import jdk.graal.compiler.nodes.NodeView;
+import jdk.graal.compiler.nodes.StructuredGraph;
+import jdk.graal.compiler.nodes.calc.AddNode;
+import jdk.graal.compiler.nodes.memory.ReadNode;
+import jdk.graal.compiler.test.AddModules;
+import jdk.graal.compiler.vector.architecture.VectorArchitecture;
+import jdk.graal.compiler.vector.architecture.VectorLoweringProvider;
+import jdk.graal.compiler.vector.nodes.simd.SimdBroadcastNode;
+import jdk.graal.compiler.vector.nodes.simd.SimdStamp;
+import jdk.incubator.vector.IntVector;
+import jdk.vm.ci.meta.JavaKind;
+
+/** Basic testing for intrinsification of Vector API operations. */
+@AddModules("jdk.incubator.vector")
+public class VectorAPISmokeTest extends GraalCompilerTest {
+
+    public static int[] testSnippet(int[] ints, int x, int[] output) {
+        IntVector.fromArray(IntVector.SPECIES_PREFERRED, ints, 0).add(x).intoArray(output, 0);
+        return output;
+    }
+
+    @Override
+    protected void checkLowTierGraph(StructuredGraph graph) {
+        /* Expect a SIMD load, a SIMD broadcast, and a SIMD add. */
+        int simdLoads = graph.getNodes().filter(ReadNode.class).filter(read -> ((ReadNode) read).stamp(NodeView.DEFAULT) instanceof SimdStamp).count();
+        Assert.assertEquals("simd loads", 1, simdLoads);
+        int simdBroadcasts = graph.getNodes().filter(SimdBroadcastNode.class).count();
+        Assert.assertEquals("simd broadcasts", 1, simdBroadcasts);
+        int simdAdds = graph.getNodes().filter(AddNode.class).filter(add -> ((AddNode) add).stamp(NodeView.DEFAULT) instanceof SimdStamp).count();
+        Assert.assertEquals("simd adds", 1, simdAdds);
+    }
+
+    @Test
+    public void runTest() {
+        int vectorLength = IntVector.SPECIES_PREFERRED.length();
+        VectorArchitecture vectorArch = ((VectorLoweringProvider) getProviders().getLowerer()).getVectorArchitecture();
+        Assume.assumeTrue("test needs minimal vectorization support",
+                        vectorArch.getSupportedVectorArithmeticLength(StampFactory.forKind(JavaKind.Int), vectorLength, IntegerStamp.OPS.getAdd()) == vectorLength);
+
+        int[] ints = new int[vectorLength];
+        for (int i = 0; i < ints.length; i++) {
+            ints[i] = i;
+        }
+        ArgSupplier output = () -> new int[ints.length];
+        test("testSnippet", ints, 7, output);
+    }
+}

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/aarch64/AArch64SuitesCreator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,9 +24,19 @@
  */
 package jdk.graal.compiler.core.aarch64;
 
+import java.util.ListIterator;
+
+import jdk.graal.compiler.core.common.GraalOptions;
 import jdk.graal.compiler.java.DefaultSuitesCreator;
 import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins;
+import jdk.graal.compiler.options.OptionValues;
+import jdk.graal.compiler.phases.BasePhase;
+import jdk.graal.compiler.phases.common.DeadCodeEliminationPhase;
 import jdk.graal.compiler.phases.tiers.CompilerConfiguration;
+import jdk.graal.compiler.phases.tiers.LowTierContext;
+import jdk.graal.compiler.phases.tiers.Suites;
+import jdk.graal.compiler.vector.phases.aarch64.AArch64VectorLoweringPhase;
+import jdk.vm.ci.code.Architecture;
 
 public abstract class AArch64SuitesCreator extends DefaultSuitesCreator {
 
@@ -37,4 +47,17 @@ public AArch64SuitesCreator(CompilerConfiguration compilerConfiguration, Plugins
     public AArch64SuitesCreator(CompilerConfiguration compilerConfiguration) {
         super(compilerConfiguration);
     }
+
+    @Override
+    public Suites createSuites(OptionValues options, Architecture arch) {
+        Suites suites = super.createSuites(options, arch);
+        if (GraalOptions.TargetVectorLowering.getValue(options)) {
+            ListIterator<BasePhase<? super LowTierContext>> position = suites.getLowTier().findPhase(DeadCodeEliminationPhase.class);
+            if (position != null) {
+                position.previous();
+                position.add(new AArch64VectorLoweringPhase());
+            }
+        }
+        return suites;
+    }
 }

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64SuitesCreator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,17 +26,19 @@
 
 import java.util.ListIterator;
 
+import jdk.graal.compiler.core.common.GraalOptions;
 import jdk.graal.compiler.java.DefaultSuitesCreator;
 import jdk.graal.compiler.lir.amd64.phases.StackMoveOptimizationPhase;
 import jdk.graal.compiler.lir.phases.LIRSuites;
 import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins;
 import jdk.graal.compiler.options.OptionValues;
 import jdk.graal.compiler.phases.BasePhase;
+import jdk.graal.compiler.phases.common.DeadCodeEliminationPhase;
 import jdk.graal.compiler.phases.common.UseTrappingNullChecksPhase;
 import jdk.graal.compiler.phases.tiers.CompilerConfiguration;
 import jdk.graal.compiler.phases.tiers.LowTierContext;
 import jdk.graal.compiler.phases.tiers.Suites;
-
+import jdk.graal.compiler.vector.phases.amd64.AMD64VectorLoweringPhase;
 import jdk.vm.ci.code.Architecture;
 
 public abstract class AMD64SuitesCreator extends DefaultSuitesCreator {
@@ -57,6 +59,13 @@ public Suites createSuites(OptionValues options, Architecture arch) {
             position.previous();
             position.add(new UseTrappingDivPhase());
         }
+        if (GraalOptions.TargetVectorLowering.getValue(options)) {
+            position = suites.getLowTier().findPhase(DeadCodeEliminationPhase.class);
+            if (position != null) {
+                position.previous();
+                position.add(new AMD64VectorLoweringPhase());
+            }
+        }
         return suites;
     }
 

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/common/GraalOptions.java

@@ -347,4 +347,7 @@ public final class GraalOptions {
     @Option(help = "The maximum number of profiled types that will be used when compiling a profiled type check. " +
                     "Note that TypeCheckMinProfileHitProbability also influences whether profiling info is used in compiled type checks.", type = OptionType.Debug)
     public static final OptionKey<Integer> TypeCheckMaxHints = new OptionKey<>(2);
+
+    @Option(help = "Enables target-specific lowering and legalization of SIMD operations. Required for SIMD code generation.", type = OptionType.Debug)
+    public static final OptionKey<Boolean> TargetVectorLowering = new OptionKey<>(true);
 }

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/CEOptimization.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -52,6 +52,8 @@
 import jdk.graal.compiler.phases.common.UseTrappingNullChecksPhase;
 import jdk.graal.compiler.phases.common.inlining.InliningPhase;
 import jdk.graal.compiler.phases.schedule.SchedulePhase;
+import jdk.graal.compiler.vector.replacements.vectorapi.VectorAPIExpansionPhase;
+import jdk.graal.compiler.vector.replacements.vectorapi.VectorAPIIntrinsics;
 import jdk.graal.compiler.virtual.phases.ea.PartialEscapePhase;
 import jdk.graal.compiler.virtual.phases.ea.ReadEliminationPhase;
 
@@ -310,7 +312,16 @@ public enum CEOptimization {
      *
      * This phase is enabled by default.
      */
-    BoxNodeOptimization(null, BoxNodeOptimizationPhase.class);
+    BoxNodeOptimization(null, BoxNodeOptimizationPhase.class),
+
+    /**
+     * {@link VectorAPIExpansionPhase} lowers Java Vector API (JEP 338) operations to corresponding
+     * SIMD code for more efficent execution.
+     *
+     * This phase is enabled by default and can be disabled with
+     * {@link jdk.graal.compiler.vector.replacements.vectorapi.VectorAPIIntrinsics.Options#OptimizeVectorAPI}.
+     */
+    VectorAPIOptimization(VectorAPIIntrinsics.Options.OptimizeVectorAPI, VectorAPIExpansionPhase.class);
 
     private final OptionKey<?> option;
     private final Class<? extends BasePhase<?>> optimization;

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/HighTier.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -48,6 +48,8 @@
 import jdk.graal.compiler.phases.common.inlining.InliningPhase;
 import jdk.graal.compiler.phases.common.inlining.policy.GreedyInliningPolicy;
 import jdk.graal.compiler.phases.tiers.HighTierContext;
+import jdk.graal.compiler.vector.replacements.vectorapi.VectorAPIExpansionPhase;
+import jdk.graal.compiler.vector.replacements.vectorapi.VectorAPIIntrinsics;
 import jdk.graal.compiler.virtual.phases.ea.FinalPartialEscapePhase;
 import jdk.graal.compiler.virtual.phases.ea.ReadEliminationPhase;
 
@@ -108,6 +110,10 @@ public HighTier(OptionValues options) {
             appendPhase(new FinalPartialEscapePhase(true, canonicalizer, null, options));
         }
 
+        if (VectorAPIIntrinsics.intrinsificationSupported(options)) {
+            appendPhase(new VectorAPIExpansionPhase(canonicalizer));
+        }
+
         if (GraalOptions.OptReadElimination.getValue(options)) {
             appendPhase(new ReadEliminationPhase(canonicalizer));
         }

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotBackend.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -75,9 +75,17 @@
 import jdk.graal.compiler.lir.framemap.FrameMapBuilder;
 import jdk.graal.compiler.lir.gen.LIRGenerationResult;
 import jdk.graal.compiler.lir.gen.LIRGeneratorTool;
+import jdk.graal.compiler.lir.gen.MoveFactory;
 import jdk.graal.compiler.nodes.StructuredGraph;
 import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool;
+import jdk.graal.compiler.vector.lir.VectorLIRGeneratorTool;
+import jdk.graal.compiler.vector.lir.aarch64.AArch64VectorArithmeticLIRGenerator;
+import jdk.graal.compiler.vector.lir.aarch64.AArch64VectorMoveFactory;
+import jdk.graal.compiler.vector.lir.aarch64.AArch64VectorNodeMatchRules;
+import jdk.graal.compiler.vector.lir.hotspot.aarch64.AArch64HotSpotSimdLIRKindTool;
+import jdk.graal.compiler.vector.lir.hotspot.aarch64.AArch64HotSpotVectorLIRGenerator;
 import jdk.internal.misc.Unsafe;
+import jdk.vm.ci.aarch64.AArch64;
 import jdk.vm.ci.aarch64.AArch64Kind;
 import jdk.vm.ci.code.CallingConvention;
 import jdk.vm.ci.code.CompilationRequest;
@@ -96,9 +104,11 @@
  * HotSpot AArch64 specific backend.
  */
 public class AArch64HotSpotBackend extends HotSpotHostBackend implements LIRGenerationProvider {
+    protected final boolean neonSupported;
 
     public AArch64HotSpotBackend(GraalHotSpotVMConfig config, HotSpotGraalRuntimeProvider runtime, HotSpotProviders providers) {
         super(config, runtime, providers);
+        neonSupported = ((AArch64) providers.getCodeCache().getTarget().arch).getFeatures().contains(AArch64.CPUFeature.ASIMD);
     }
 
     @Override
@@ -110,12 +120,26 @@ protected FrameMapBuilder newFrameMapBuilder(RegisterConfig registerConfig, Stub
 
     @Override
     public LIRGeneratorTool newLIRGenerator(LIRGenerationResult lirGenRes) {
-        return new AArch64HotSpotLIRGenerator(getProviders(), config, lirGenRes);
+        if (neonSupported) {
+            return new AArch64HotSpotVectorLIRGenerator(
+                            new AArch64HotSpotSimdLIRKindTool(),
+                            new AArch64VectorArithmeticLIRGenerator(null),
+                            new AArch64VectorMoveFactory(new AArch64HotSpotMoveFactory(), new MoveFactory.BackupSlotProvider(lirGenRes.getFrameMapBuilder())),
+                            getProviders(),
+                            config,
+                            lirGenRes);
+        } else {
+            return new AArch64HotSpotLIRGenerator(getProviders(), config, lirGenRes);
+        }
     }
 
     @Override
     public NodeLIRBuilderTool newNodeLIRBuilder(StructuredGraph graph, LIRGeneratorTool lirGen) {
-        return new AArch64HotSpotNodeLIRBuilder(graph, lirGen, new AArch64NodeMatchRules(lirGen));
+        if (lirGen.getArithmetic() instanceof VectorLIRGeneratorTool) {
+            return new AArch64HotSpotNodeLIRBuilder(graph, lirGen, new AArch64VectorNodeMatchRules(lirGen));
+        } else {
+            return new AArch64HotSpotNodeLIRBuilder(graph, lirGen, new AArch64NodeMatchRules(lirGen));
+        }
     }
 
     @Override

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotBackendFactory.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2025, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2018, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -31,7 +31,6 @@
 import java.util.List;
 
 import jdk.graal.compiler.core.aarch64.AArch64AddressLoweringByUse;
-import jdk.graal.compiler.core.aarch64.AArch64LIRKindTool;
 import jdk.graal.compiler.core.aarch64.AArch64SuitesCreator;
 import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig;
 import jdk.graal.compiler.hotspot.HotSpotBackendFactory;
@@ -58,6 +57,8 @@
 import jdk.graal.compiler.phases.tiers.CompilerConfiguration;
 import jdk.graal.compiler.replacements.aarch64.AArch64GraphBuilderPlugins;
 import jdk.graal.compiler.serviceprovider.ServiceProvider;
+import jdk.graal.compiler.vector.architecture.aarch64.VectorAArch64;
+import jdk.graal.compiler.vector.lir.hotspot.aarch64.AArch64HotSpotSimdLIRKindTool;
 import jdk.vm.ci.aarch64.AArch64;
 import jdk.vm.ci.code.Register;
 import jdk.vm.ci.code.RegisterConfig;
@@ -66,6 +67,7 @@
 import jdk.vm.ci.hotspot.HotSpotConstantReflectionProvider;
 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime;
 import jdk.vm.ci.hotspot.aarch64.AArch64HotSpotRegisterConfig;
+import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.MetaAccessProvider;
 import jdk.vm.ci.meta.Value;
 
@@ -134,15 +136,17 @@ protected HotSpotHostForeignCallsProvider createForeignCalls(HotSpotJVMCIRuntime
     protected HotSpotSuitesProvider createSuites(GraalHotSpotVMConfig config, HotSpotGraalRuntimeProvider runtime, CompilerConfiguration compilerConfiguration, Plugins plugins,
                     HotSpotRegistersProvider registers, HotSpotReplacementsImpl replacements, OptionValues options) {
         AArch64SuitesCreator suitesCreator = new AArch64HotSpotSuitesCreator(compilerConfiguration, plugins);
-        BasePhase<CoreProviders> addressLoweringPhase = new AddressLoweringByUsePhase(new AArch64AddressLoweringByUse(new AArch64LIRKindTool(), true));
+        BasePhase<CoreProviders> addressLoweringPhase = new AddressLoweringByUsePhase(new AArch64AddressLoweringByUse(new AArch64HotSpotSimdLIRKindTool(), true));
         return new AddressLoweringHotSpotSuitesProvider(suitesCreator, config, runtime, addressLoweringPhase);
     }
 
     @Override
     protected HotSpotLoweringProvider createLowerer(HotSpotGraalRuntimeProvider graalRuntime, MetaAccessProvider metaAccess, HotSpotHostForeignCallsProvider foreignCalls,
                     HotSpotRegistersProvider registers, HotSpotConstantReflectionProvider constantReflection, HotSpotPlatformConfigurationProvider platformConfig,
                     HotSpotMetaAccessExtensionProvider metaAccessExtensionProvider, TargetDescription target) {
-        return new AArch64HotSpotLoweringProvider(graalRuntime, metaAccess, foreignCalls, registers, constantReflection, platformConfig, metaAccessExtensionProvider, target);
+        VectorAArch64 varch = new VectorAArch64((AArch64) target.arch, metaAccess.getArrayIndexScale(JavaKind.Object), graalRuntime.getVMConfig().useCompressedOops,
+                        graalRuntime.getVMConfig().objectAlignment, graalRuntime.getVMConfig().maxVectorSize);
+        return new AArch64HotSpotLoweringProvider(graalRuntime, metaAccess, foreignCalls, registers, constantReflection, platformConfig, metaAccessExtensionProvider, target, varch);
     }
 
     @Override