[GR-65623] Implement layer compatibility check for class and module-path entries.

Author: Tudor-Stefan Magirescu

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/util/Digest.java

@@ -102,8 +102,10 @@ public static String digest(byte[] bytes, int offset, int length) {
      * new byte array.
      */
     public static byte[] digestAsByteArray(byte[] bytes, int offset, int length) {
-        LongLong hash = MurmurHash3_x64_128(bytes, offset, length, HASH_SEED);
+        return longLongToByteArray(MurmurHash3_x64_128(bytes, offset, length, HASH_SEED));
+    }
 
+    private static byte[] longLongToByteArray(LongLong hash) {
         byte[] array = new byte[DIGEST_SIZE];
         encodeBase62(hash.l1, array, 0);
         encodeBase62(hash.l2, array, BASE62_DIGITS_PER_LONG);
@@ -281,4 +283,20 @@ private static long fmix64(long input) {
 
         return k;
     }
+
+    public static final class DigestBuilder {
+        private LongLong digest;
+
+        public DigestBuilder() {
+            digest = new LongLong(0L, 0L);
+        }
+
+        public void update(byte[] input) {
+            digest = MurmurHash3_x64_128(input, 0, input.length, digest.l1 ^ digest.l2);
+        }
+
+        public byte[] digest() {
+            return longLongToByteArray(digest);
+        }
+    }
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JRTSupport.java

@@ -29,6 +29,7 @@
 import java.net.URL;
 import java.net.URLConnection;
 import java.nio.ByteBuffer;
+import java.nio.file.FileSystem;
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -181,6 +182,13 @@ protected URLConnection openConnection(URL url) throws IOException {
 final class Target_jdk_internal_jrtfs_JrtFileSystemProvider_JRTDisabled {
 }
 
+@TargetClass(className = "jdk.internal.jrtfs.JrtFileSystemProvider", onlyWith = JRTEnabled.class)
+final class Target_jdk_internal_jrtfs_JrtFileSystemProvider_BuildTime {
+    @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Reset)//
+    @Alias//
+    volatile FileSystem theFileSystem;
+}
+
 // endregion Disable jimage/jrtfs
 
 @TargetClass(className = "jdk.internal.jimage.BasicImageReader")

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java

@@ -24,6 +24,8 @@
  */
 package com.oracle.svm.hosted;
 
+import static jdk.graal.compiler.util.Digest.DigestBuilder;
+
 import java.io.File;
 import java.io.IOException;
 import java.lang.module.Configuration;
@@ -36,9 +38,12 @@
 import java.lang.reflect.Method;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.net.URL;
 import java.nio.channels.ClosedByInterruptException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
+import java.nio.file.FileVisitOption;
 import java.nio.file.FileVisitResult;
 import java.nio.file.FileVisitor;
 import java.nio.file.Files;
@@ -52,6 +57,8 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Deque;
+import java.util.EnumSet;
+import java.util.Enumeration;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
@@ -67,9 +74,12 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.LongAdder;
 import java.util.function.BiConsumer;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
 import java.util.stream.Collector;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import java.util.zip.ZipFile;
 
 import org.graalvm.collections.EconomicMap;
 import org.graalvm.collections.EconomicSet;
@@ -116,6 +126,7 @@ public final class NativeImageClassLoaderSupport {
     private final List<Path> buildmp;
 
     private final Set<Path> imageProvidedJars;
+    private PathDigests pathDigests;
 
     private final EconomicMap<URI, EconomicSet<String>> classes;
     private final EconomicMap<URI, EconomicSet<String>> packages;
@@ -300,6 +311,47 @@ public NativeImageClassLoader getClassLoader() {
         return classLoader;
     }
 
+    public Optional<PathDigests> getPathDigests(boolean free) {
+        Optional<PathDigests> res = Optional.ofNullable(pathDigests);
+        if (free) {
+            pathDigests = null;
+        }
+        return res;
+    }
+
+    public void initializePathDigests(Path digestIgnoreRelativePath) {
+        List<Path> digestIgnorePaths = new ArrayList<>();
+        if (digestIgnoreRelativePath != null) {
+            try {
+                Enumeration<URL> urls = classLoader.getResources(digestIgnoreRelativePath.toString());
+                while (urls.hasMoreElements()) {
+                    URI uri = urls.nextElement().toURI();
+                    if ("jar".equalsIgnoreCase(uri.getScheme())) {
+                        String uriString = uri.toString();
+                        String fileUriString = uriString.substring("jar:".length(), uriString.indexOf("!/"));
+                        uri = URI.create(fileUriString);
+                    }
+                    digestIgnorePaths.add(Path.of(uri));
+                }
+            } catch (URISyntaxException | IOException e) {
+                throw UserError.abort("Error while looking for %s in %s", digestIgnoreRelativePath, classLoader);
+            }
+        }
+
+        pathDigests = new PathDigests(filterIgnoredPathEntries(imagecp, digestIgnorePaths), filterIgnoredPathEntries(imagemp, digestIgnorePaths));
+    }
+
+    private List<Path> filterIgnoredPathEntries(List<Path> pathEntries, List<Path> digestIgnorePaths) {
+        return pathEntries.stream().filter(pathEntry -> {
+            for (Path p : digestIgnorePaths) {
+                if (p.startsWith(pathEntry)) {
+                    return false;
+                }
+            }
+            return true;
+        }).toList();
+    }
+
     public LibGraalLoader getLibGraalLoader() {
         VMError.guarantee(libGraalLoader != null, "Invalid access to libGraalLoader before getting set up");
         return libGraalLoader.orElse(null);
@@ -932,13 +984,18 @@ private void initModule(ModuleReference moduleReference, boolean moduleRequiresI
                 if (ModuleLayer.boot().equals(module.getLayer())) {
                     builderURILocations.add(container);
                 }
+                final boolean isInImageModulePathOfLayeredBuild = pathDigests != null && pathDigests.mpDigests.containsKey(container);
+                final boolean isJar = ClasspathUtils.isJar(Path.of(container));
                 moduleReader.list().forEach(moduleResource -> {
                     char fileSystemSeparatorChar = '/';
                     String className = extractClassName(moduleResource, fileSystemSeparatorChar);
                     if (className != null) {
                         currentlyProcessedEntry = moduleReferenceLocation + fileSystemSeparatorChar + moduleResource;
                         executor.execute(() -> handleClassFileName(container, module, className, includeUnconditionally, moduleRequiresInit, preserveModule));
                     }
+                    if (isInImageModulePathOfLayeredBuild) {
+                        executor.execute(() -> PathDigests.storePathFileDigest(container, moduleResource, isJar, pathDigests.mpDigests));
+                    }
                     entriesProcessed.increment();
                 });
             } catch (IOException e) {
@@ -962,7 +1019,7 @@ private void loadClassesFromPath(Path path) {
                     }
                     if (probeJarFileSystem != null) {
                         try (FileSystem jarFileSystem = probeJarFileSystem) {
-                            loadClassesFromPath(container, jarFileSystem.getPath("/"), null, Collections.emptySet(), includeUnconditionally, includeAllMetadata);
+                            loadClassesFromPath(container, jarFileSystem.getPath("/"), null, Collections.emptySet(), includeUnconditionally, includeAllMetadata, true);
                         }
                     }
                 } catch (ClosedByInterruptException ignored) {
@@ -973,20 +1030,21 @@ private void loadClassesFromPath(Path path) {
             } else {
                 URI container = path.toUri();
                 loadClassesFromPath(container, path, ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES_ROOT, ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES, includeUnconditionally,
-                                includeAllMetadata);
+                                includeAllMetadata, false);
             }
         }
 
         private static final String CLASS_EXTENSION = ".class";
 
-        private void loadClassesFromPath(URI container, Path root, Path excludeRoot, Set<Path> excludes, boolean includeUnconditionally, boolean includeAllMetadata) {
+        private void loadClassesFromPath(URI container, Path root, Path excludeRoot, Set<Path> excludes, boolean includeUnconditionally, boolean includeAllMetadata, boolean isJar) {
             boolean useFilter = root.equals(excludeRoot);
             if (useFilter) {
                 String excludesStr = excludes.stream().map(Path::toString).collect(Collectors.joining(", "));
                 LogUtils.warning("Using directory %s on classpath is discouraged. Reading classes/resources from directories %s will be suppressed.", excludeRoot, excludesStr);
             }
             FileVisitor<Path> visitor = new SimpleFileVisitor<>() {
                 private final char fileSystemSeparatorChar = root.getFileSystem().getSeparator().charAt(0);
+                private final boolean isInImageClassPathOfLayeredBuild = pathDigests != null && pathDigests.getCpDigests().containsKey(container);
 
                 @Override
                 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
@@ -1007,6 +1065,9 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
                         currentlyProcessedEntry = file.toUri().toString();
                         executor.execute(() -> handleClassFileName(container, null, className, includeUnconditionally, true, includeAllMetadata));
                     }
+                    if (isInImageClassPathOfLayeredBuild) {
+                        executor.execute(() -> PathDigests.storePathFileDigest(container, fileName, isJar, pathDigests.cpDigests));
+                    }
                     entriesProcessed.increment();
                     return FileVisitResult.CONTINUE;
                 }
@@ -1019,7 +1080,7 @@ public FileVisitResult visitFileFailed(Path file, IOException exc) {
             };
 
             try {
-                Files.walkFileTree(root, visitor);
+                Files.walkFileTree(root, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, visitor);
             } catch (IOException ex) {
                 throw VMError.shouldNotReachHere(ex);
             }
@@ -1372,4 +1433,54 @@ public Set<IncludeOptionsSupport.PackageOptionValue> packages() {
             return packages.keySet();
         }
     }
+
+    public static final class PathDigests {
+        private final EconomicMap<URI, List<String>> cpDigests = EconomicMap.create();
+        private final EconomicMap<URI, List<String>> mpDigests = EconomicMap.create();
+
+        private PathDigests(List<Path> imagecp, List<Path> imagemp) {
+            imagecp.stream()
+                            .map(Path::toUri)
+                            .forEach(path -> cpDigests.put(path, new ArrayList<>()));
+            imagemp.stream()
+                            .map(Path::toUri)
+                            .forEach(path -> mpDigests.put(path, new ArrayList<>()));
+        }
+
+        private static void storePathFileDigest(URI container, String resource, boolean isJar, EconomicMap<URI, List<String>> digests) {
+            byte[] fileContent;
+            try {
+                if (isJar) {
+                    try (JarFile jarFile = new JarFile(new File(container), true, ZipFile.OPEN_READ, JarFile.runtimeVersion())) {
+                        JarEntry jarEntry = jarFile.getJarEntry(resource);
+                        fileContent = jarFile.getInputStream(jarEntry).readAllBytes();
+                    }
+                } else {
+                    Path resourcePath = Path.of(container).resolve(resource);
+                    if (!resourcePath.toFile().isFile()) {
+                        return;
+                    }
+                    fileContent = Files.readAllBytes(resourcePath);
+                }
+            } catch (IOException e) {
+                throw UserError.abort("Image builder cannot read file: " + resource);
+            }
+
+            DigestBuilder db = new DigestBuilder();
+            db.update(fileContent);
+            db.update(resource.getBytes(StandardCharsets.UTF_8));
+            List<String> containerDigests = digests.get(container);
+            synchronized (containerDigests) {
+                containerDigests.add(new String(db.digest(), StandardCharsets.UTF_8));
+            }
+        }
+
+        public EconomicMap<URI, List<String>> getCpDigests() {
+            return cpDigests;
+        }
+
+        public EconomicMap<URI, List<String>> getMpDigests() {
+            return mpDigests;
+        }
+    }
 }

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/driver/LayerOptionsSupport.java

@@ -35,7 +35,9 @@
 public class LayerOptionsSupport extends IncludeOptionsSupport {
 
     public record LayerOption(Path fileName, ExtendedOption[] extendedOptions) {
-        /** Split a layer option into its components. */
+        /**
+         * Split a layer option into its components.
+         */
         public static LayerOption parse(String layerOptionValue) {
             VMError.guarantee(!layerOptionValue.isEmpty());
             // Given an argument of form layer-file.nil,module=m1,package=p1

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java

@@ -187,6 +187,7 @@ public Function<Class<?>, SingletonTrait[]> getSingletonTraitInjector() {
      */
     public static void processLayerOptions(EconomicMap<OptionKey<?>, Object> values, NativeImageClassLoaderSupport classLoaderSupport) {
         OptionValues hostedOptions = new OptionValues(values);
+        Path digestIgnorePath = null;
 
         if (isLayerCreateOptionEnabled(hostedOptions)) {
             ValueWithOrigin<String> valueWithOrigin = getLayerCreateValueWithOrigin(hostedOptions);
@@ -205,7 +206,12 @@ public static void processLayerOptions(EconomicMap<OptionKey<?>, Object> values,
             classLoaderSupport.setLayerFile(layerFile);
 
             NativeImageClassLoaderSupport.IncludeSelectors layerSelectors = classLoaderSupport.getLayerSelectors();
+            IncludeOptionsSupport.ExtendedOption digestIgnoreExtendedOption = new IncludeOptionsSupport.ExtendedOption("digest-ignore", null);
             for (IncludeOptionsSupport.ExtendedOption option : layerOption.extendedOptions()) {
+                if (option.equals(digestIgnoreExtendedOption)) {
+                    digestIgnorePath = valueWithOrigin.origin().location();
+                    continue;
+                }
                 IncludeOptionsSupport.parseIncludeSelector(layerCreateArg, valueWithOrigin, layerSelectors, option, layerCreatePossibleOptions());
             }
 
@@ -238,6 +244,10 @@ public static void processLayerOptions(EconomicMap<OptionKey<?>, Object> values,
             SubstrateOptions.ApplicationLayerInitializedClasses.update(values, Module.class.getName());
             setOptionIfHasNotBeenSet(values, SubstrateOptions.ConcealedOptions.RelativeCodePointers, true);
         }
+
+        if (isLayerCreateOptionEnabled(hostedOptions) || isLayerUseOptionEnabled(hostedOptions)) {
+            classLoaderSupport.initializePathDigests(digestIgnorePath);
+        }
     }
 
     private static void setOptionIfHasNotBeenSet(EconomicMap<OptionKey<?>, Object> values, HostedOptionKey<Boolean> option, boolean boxedValue) {
@@ -285,7 +295,7 @@ private static String getLayerCreateValue(ValueWithOrigin<String> valueWithOrigi
         return String.join(",", OptionUtils.resolveOptionValuesRedirection(SubstrateOptions.LayerCreate, valueWithOrigin));
     }
 
-    private static boolean isLayerUseOptionEnabled(OptionValues values) {
+    public static boolean isLayerUseOptionEnabled(OptionValues values) {
         if (SubstrateOptions.LayerUse.hasBeenSet(values)) {
             return !getLayerUseValue(values).toString().isEmpty();
         }