Add JDK Api extractor tool to help with entitlement reviews

libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/Utils.java

@@ -24,12 +24,22 @@
 
 public class Utils {
 
+    // TODO Currently ServerProcessBuilder is using --add-modules=ALL-MODULE-PATH, should this rather
+    // reflect below excludes (except for java.desktop which requires a special handling)?
+    // internal and incubator modules are also excluded
     private static final Set<String> EXCLUDED_MODULES = Set.of(
         "java.desktop",
         "jdk.jartool",
         "jdk.jdi",
         "java.security.jgss",
-        "jdk.jshell"
+        "jdk.jshell",
+        "jdk.jcmd",
+        "jdk.hotspot.agent",
+        "jdk.jfr",
+        "jdk.javadoc",
+        // "jdk.jpackage", // Do we want to include this?
+        // "jdk.jlink", // Do we want to include this?
+        "jdk.localedata" // noise, change here are not interesting
     );
 
     private static Map<String, Set<String>> findModuleExports(FileSystem fs) throws IOException {
@@ -70,7 +80,9 @@ public static void walkJdkModules(JdkModuleConsumer c) throws IOException {
 
             for (var kv : modules.entrySet()) {
                 var moduleName = kv.getKey();
-                if (Utils.EXCLUDED_MODULES.contains(moduleName) == false) {
+                if (Utils.EXCLUDED_MODULES.contains(moduleName) == false
+                    && moduleName.contains(".internal.") == false
+                    && moduleName.contains(".incubator.") == false) {
                     var thisModuleExports = moduleExports.get(moduleName);
                     c.accept(moduleName, kv.getValue(), thisModuleExports);
                 }

libs/entitlement/tools/jdk-api-extractor/README.md

@@ -0,0 +1,20 @@
+This tool scans the JDK on which it is running to extract its public accessible API.
+That is:
+- public methods (including constructors) of public, exported classes as well as protected methods of these if not final.
+- internal implementations (overwrites) of above.
+
+The output of this tool is meant to be diffed against the output for another JDK
+version to identify changes that need to be reviewed for entitlements.
+
+Usage example:
+```bash
+./gradlew :libs:entitlement:tools:jdk-api-extractor:run -Druntime.java=24 --args="api-jdk24.tsv"
+./gradlew :libs:entitlement:tools:jdk-api-extractor:run -Druntime.java=25 --args="api-jdk25.tsv"
+diff libs/entitlement/tools/jdk-api-extractor/api-jdk24.tsv libs/entitlement/tools/jdk-api-extractor/api-jdk25.tsv
+```
+
+To review the diff of deprecations, use `--deprecations-only` as 2nd argument.
+
+```bash
+./gradlew :libs:entitlement:tools:jdk-api-extractor:run -Druntime.java=24 --args="deprecations-jdk24.tsv --deprecations-only"
+```

libs/entitlement/tools/jdk-api-extractor/build.gradle

@@ -0,0 +1,68 @@
+plugins {
+  id 'application'
+}
+
+apply plugin: 'elasticsearch.build'
+
+tasks.named("dependencyLicenses").configure {
+  mapping from: /asm-.*/, to: 'asm'
+}
+
+group = 'org.elasticsearch.entitlement.tools'
+
+ext {
+  javaMainClass = "org.elasticsearch.entitlement.tools.jdkapi.JdkApiExtractor"
+}
+
+application {
+  mainClass.set(javaMainClass)
+  applicationDefaultJvmArgs = [
+    '--add-exports', 'java.base/sun.security.util=ALL-UNNAMED',
+    '--add-opens', 'java.base/java.lang=ALL-UNNAMED',
+    '--add-opens', 'java.base/java.net=ALL-UNNAMED',
+    '--add-opens', 'java.base/java.net.spi=ALL-UNNAMED',
+    '--add-opens', 'java.base/java.util.concurrent=ALL-UNNAMED',
+    '--add-opens', 'java.base/javax.crypto=ALL-UNNAMED',
+    '--add-opens', 'java.base/javax.security.auth=ALL-UNNAMED',
+    '--add-opens', 'java.base/jdk.internal.logger=ALL-UNNAMED',
+    '--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED',
+    '--add-opens', 'jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED',
+    '--add-opens', 'java.logging/java.util.logging=ALL-UNNAMED',
+    '--add-opens', 'java.logging/sun.util.logging.internal=ALL-UNNAMED',
+    '--add-opens', 'java.naming/javax.naming.ldap.spi=ALL-UNNAMED',
+    '--add-opens', 'java.rmi/sun.rmi.runtime=ALL-UNNAMED',
+    '--add-opens', 'jdk.dynalink/jdk.dynalink=ALL-UNNAMED',
+    '--add-opens', 'jdk.dynalink/jdk.dynalink.linker=ALL-UNNAMED',
+    '--add-opens', 'java.desktop/sun.awt=ALL-UNNAMED',
+    '--add-opens', 'java.sql.rowset/javax.sql.rowset.spi=ALL-UNNAMED',
+    '--add-opens', 'java.sql/java.sql=ALL-UNNAMED',
+    '--add-opens', 'java.xml.crypto/com.sun.org.apache.xml.internal.security.utils=ALL-UNNAMED'
+  ]
+}
+
+// TODO Better integrate this into the existing tooling,
+// usage of Druntime.java=24 doesn't work here at the moment out of the box
+tasks.withType(JavaExec) {
+  javaLauncher = javaToolchains.launcherFor {
+    languageVersion = JavaLanguageVersion.of(System.getProperty("runtime.java", "24"))
+  }
+}
+
+repositories {
+  mavenCentral()
+}
+
+dependencies {
+  compileOnly(project(':libs:core'))
+  implementation 'org.ow2.asm:asm:9.8'
+  implementation 'org.ow2.asm:asm-util:9.8'
+  implementation(project(':libs:entitlement:tools:common'))
+}
+
+tasks.named('forbiddenApisMain').configure {
+  replaceSignatureFiles 'jdk-signatures'
+}
+
+tasks.named("thirdPartyAudit").configure {
+  ignoreMissingClasses()
+}

libs/entitlement/tools/jdk-api-extractor/licenses/asm-LICENSE.txt

@@ -0,0 +1,26 @@
+Copyright (c) 2012 France Télécom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the name of the copyright holders nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.

libs/entitlement/tools/jdk-api-extractor/licenses/asm-NOTICE.txt

@@ -0,0 +1 @@
+

libs/entitlement/tools/jdk-api-extractor/src/main/java/org/elasticsearch/entitlement/tools/jdkapi/JdkApiExtractor.java

@@ -0,0 +1,245 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.tools.jdkapi;
+
+import org.elasticsearch.entitlement.tools.Utils;
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.MethodVisitor;
+
+import java.io.IOException;
+import java.lang.constant.ClassDesc;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Stream;
+
+import static org.objectweb.asm.Opcodes.ACC_DEPRECATED;
+import static org.objectweb.asm.Opcodes.ACC_FINAL;
+import static org.objectweb.asm.Opcodes.ACC_PROTECTED;
+import static org.objectweb.asm.Opcodes.ACC_PUBLIC;
+import static org.objectweb.asm.Opcodes.ACC_STATIC;
+import static org.objectweb.asm.Opcodes.ASM9;
+
+public class JdkApiExtractor {
+
+    // exclude both final and non-final variants of these
+    private static final Set<AccessibleMethod> EXCLUDES = Set.of(
+        new AccessibleMethod("toString", "()Ljava/lang/String;", true, false, false),
+        new AccessibleMethod("hashCode", "()I", true, false, false),
+        new AccessibleMethod("equals", "(Ljava/lang/Object;)Z", true, false, false),
+        new AccessibleMethod("close", "()V", true, false, false),
+        new AccessibleMethod("toString", "()Ljava/lang/String;", true, true, false),
+        new AccessibleMethod("hashCode", "()I", true, false, true),
+        new AccessibleMethod("equals", "(Ljava/lang/Object;)Z", true, true, false),
+        new AccessibleMethod("close", "()V", true, false, true)
+    );
+
+    public static void main(String[] args) throws IOException {
+        boolean valid = args.length == 1 || (args.length == 2 && "--deprecations-only".equals(args[1]));
+
+        if (valid == false) {
+            System.err.println("usage: <output file path> [--deprecations-only]");
+            System.exit(1);
+        }
+
+        boolean deprecationsOnly = args.length == 2 && args[1].equals("--deprecations-only");
+
+        Map<String, Set<AccessibleMethod>> accessibleImplementationsByClass = new TreeMap<>();
+        Map<String, Set<AccessibleMethod>> accessibleForOverridesByClass = new TreeMap<>();
+        Map<String, Set<AccessibleMethod>> deprecationsByClass = new TreeMap<>();
+
+        Utils.walkJdkModules((moduleName, moduleClasses, moduleExports) -> {
+            var visitor = new AccessibleClassVisitor(
+                moduleExports,
+                accessibleImplementationsByClass,
+                accessibleForOverridesByClass,
+                deprecationsByClass
+            );
+            for (var classFile : moduleClasses) {
+                try {
+                    ClassReader cr = new ClassReader(Files.newInputStream(classFile));
+                    cr.accept(visitor, 0);
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        });
+
+        writeFile(Path.of(args[0]), deprecationsOnly ? deprecationsByClass : accessibleImplementationsByClass);
+    }
+
+    private static void writeFile(Path path, Map<String, Set<AccessibleMethod>> methods) throws IOException {
+        System.out.println("Writing result for " + Runtime.version() + " to " + path.toAbsolutePath());
+        Files.write(path, () -> methods.entrySet().stream().flatMap(AccessibleMethod::toLines).iterator(), StandardCharsets.UTF_8);
+    }
+
+    record AccessibleMethod(String method, String descriptor, boolean isPublic, boolean isFinal, boolean isStatic) {
+
+        private static final String SEPARATOR = "\t";
+
+        private static final Comparator<AccessibleMethod> COMPARATOR = Comparator.comparing(AccessibleMethod::method)
+            .thenComparing(AccessibleMethod::descriptor)
+            .thenComparing(AccessibleMethod::isStatic);
+
+        CharSequence toLine(String clazz) {
+            return String.join(
+                SEPARATOR,
+                clazz,
+                method,
+                descriptor,
+                isPublic ? "PUBLIC" : "PROTECTED",
+                isStatic ? "STATIC" : "",
+                isFinal ? "FINAL" : ""
+            );
+        }
+
+        static Stream<CharSequence> toLines(Map.Entry<String, Set<AccessibleMethod>> entry) {
+            return entry.getValue().stream().map(m -> m.toLine(entry.getKey()));
+        }
+    }
+
+    static class AccessibleClassVisitor extends ClassVisitor {
+        private final Set<String> moduleExports;
+        private final Map<String, Set<AccessibleMethod>> accessibleImplementationsByClass;
+        private final Map<String, Set<AccessibleMethod>> accessibleForOverridesByClass;
+        private final Map<String, Set<AccessibleMethod>> deprecationsByClass;
+
+        private Set<AccessibleMethod> accessibleImplementations;
+        private Set<AccessibleMethod> accessibleForOverrides;
+        private Set<AccessibleMethod> deprecations;
+
+        private String className;
+        private boolean isPublicClass;
+        private boolean isFinalClass;
+        private boolean isDeprecatedClass;
+        private boolean isExported;
+
+        AccessibleClassVisitor(
+            Set<String> moduleExports,
+            Map<String, Set<AccessibleMethod>> accessibleImplementationsByClass,
+            Map<String, Set<AccessibleMethod>> accessibleForOverridesByClass,
+            Map<String, Set<AccessibleMethod>> deprecationsByClass
+        ) {
+            super(ASM9);
+            this.moduleExports = moduleExports;
+            this.accessibleImplementationsByClass = accessibleImplementationsByClass;
+            this.accessibleForOverridesByClass = accessibleForOverridesByClass;
+            this.deprecationsByClass = deprecationsByClass;
+        }
+
+        private Set<AccessibleMethod> getMethods(Map<String, Set<AccessibleMethod>> methods, String clazz) {
+            return methods.computeIfAbsent(clazz, k -> new TreeSet<>(AccessibleMethod.COMPARATOR));
+        }
+
+        @Override
+        public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) {
+            if (superName != null) {
+                visitSuperClass(superName);
+                getMethods(accessibleForOverridesByClass, name).addAll(getMethods(accessibleForOverridesByClass, superName));
+            }
+            if (interfaces != null && interfaces.length > 0) {
+                for (var interfaceName : interfaces) {
+                    visitInterface(interfaceName);
+                    getMethods(accessibleForOverridesByClass, name).addAll(getMethods(accessibleForOverridesByClass, interfaceName));
+                }
+            }
+            super.visit(version, access, name, signature, superName, interfaces);
+            this.isExported = moduleExports.contains(getPackageName(name));
+            this.className = name;
+            this.isPublicClass = (access & ACC_PUBLIC) != 0;
+            this.isFinalClass = (access & ACC_FINAL) != 0;
+            this.isDeprecatedClass = (access & ACC_DEPRECATED) != 0;
+            this.accessibleImplementations = getMethods(accessibleImplementationsByClass, name);
+            this.accessibleForOverrides = getMethods(accessibleForOverridesByClass, name);
+            this.deprecations = getMethods(deprecationsByClass, name);
+        }
+
+        @Override
+        public void visitEnd() {
+            super.visitEnd();
+            if (accessibleImplementations.isEmpty()) {
+                accessibleImplementationsByClass.remove(className);
+            }
+            if (accessibleForOverrides.isEmpty()) {
+                accessibleForOverridesByClass.remove(className);
+            }
+            if (deprecations.isEmpty()) {
+                deprecationsByClass.remove(className);
+            }
+        }
+
+        private void visitSuperClass(String superName) {
+            try {
+                ClassReader cr = new ClassReader(superName);
+                cr.accept(this, 0);
+            } catch (IOException e) {
+                System.out.println("Failed to visit super class [" + superName + "]:" + e.getMessage());
+            }
+        }
+
+        private void visitInterface(String interfaceName) {
+            try {
+                ClassReader cr = new ClassReader(interfaceName);
+                cr.accept(this, 0);
+            } catch (IOException e) {
+                System.out.println("Failed to visit interface [" + interfaceName + "]:" + e.getMessage());
+
+            }
+        }
+
+        public String getClassName() {
+            return className;
+        }
+
+        private static String getPackageName(String className) {
+            return ClassDesc.ofInternalName(className).packageName();
+        }
+
+        @Override
+        public final MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
+            MethodVisitor mv = super.visitMethod(access, name, descriptor, signature, exceptions);
+            boolean isPublic = (access & ACC_PUBLIC) != 0;
+            boolean isProtected = (access & ACC_PROTECTED) != 0;
+            boolean isFinal = (access & ACC_FINAL) != 0;
+            boolean isStatic = (access & ACC_STATIC) != 0;
+            boolean isDeprecated = (access & ACC_DEPRECATED) != 0;
+            if ((isPublic || isProtected) == false) {
+                return mv;
+            }
+
+            var method = new AccessibleMethod(name, descriptor, isPublic, isFinal, isStatic);
+            if (isPublicClass && isExported && EXCLUDES.contains(method) == false) {
+                // class is public and exported, for final classes skip non-public methods
+                if (isPublic || isFinalClass == false) {
+                    accessibleImplementations.add(method);
+                    // if not static, the method is accessible for overrides
+                    if (isStatic == false) {
+                        accessibleForOverrides.add(method);
+                    }
+                    if (isDeprecatedClass || isDeprecated) {
+                        deprecations.add(method);
+                    }
+                }
+            } else if (accessibleForOverrides.contains(method)) {
+                accessibleImplementations.add(method);
+                if (isDeprecatedClass || isDeprecated) {
+                    deprecations.add(method);
+                }
+            }
+            return mv;
+        }
+    }
+}