8278087: Deserialization filter and filter factory property error reporting under specified

Backport-of: f90425a

Author: Roger Riggs

src/java.base/share/classes/java/io/ObjectInputFilter.java

@@ -27,7 +27,6 @@
 
 import jdk.internal.access.SharedSecrets;
 import jdk.internal.util.StaticProperty;
-import sun.security.action.GetBooleanAction;
 
 import java.lang.reflect.InvocationTargetException;
 import java.security.AccessController;
@@ -523,10 +522,15 @@ enum Status {
      * {@systemProperty jdk.serialFilter}, its value is used to configure the filter.
      * If the system property is not defined, and the {@link java.security.Security} property
      * {@code jdk.serialFilter} is defined then it is used to configure the filter.
-     * The filter is created as if {@link #createFilter(String) createFilter} is called;
-     * if the filter string is invalid, an {@link ExceptionInInitializerError} is thrown.
-     * Otherwise, the filter is not configured during initialization and
-     * can be set with {@link #setSerialFilter(ObjectInputFilter) Config.setSerialFilter}.
+     * The filter is created as if {@link #createFilter(String) createFilter} is called,
+     * if the filter string is invalid the initialization fails and subsequent attempts to
+     * {@linkplain Config#getSerialFilter() get the filter}, {@linkplain Config#setSerialFilter set a filter},
+     * or create an {@linkplain ObjectInputStream#ObjectInputStream(InputStream) ObjectInputStream}
+     * throw {@link IllegalStateException}. Deserialization is not possible with an
+     * invalid serial filter.
+     * If the system property {@code jdk.serialFilter} or the {@link java.security.Security}
+     * property {@code jdk.serialFilter} is not set the filter can be set with
+     * {@link #setSerialFilter(ObjectInputFilter) Config.setSerialFilter}.
      * Setting the {@code jdk.serialFilter} with {@link System#setProperty(String, String)
      * System.setProperty} <em>does not set the filter</em>.
      * The syntax for the property value is the same as for the
@@ -545,9 +549,12 @@ enum Status {
      * <p>The class must be public, must have a public zero-argument constructor, implement the
      * {@link BinaryOperator {@literal BinaryOperator<ObjectInputFilter>}} interface, provide its implementation and
      * be accessible via the {@linkplain ClassLoader#getSystemClassLoader() application class loader}.
-     * If the filter factory constructor is not invoked successfully, an {@link ExceptionInInitializerError}
-     * is thrown and subsequent use of the filter factory for deserialization fails with
-     * {@link IllegalStateException}.
+     * If the filter factory constructor is not invoked successfully subsequent attempts to
+     * {@linkplain Config#getSerialFilterFactory() get the factory},
+     * {@linkplain Config#setSerialFilterFactory(BinaryOperator) set the factory}, or create an
+     * {@link ObjectInputStream#ObjectInputStream(InputStream) ObjectInputStream}
+     * throw {@link IllegalStateException}. Deserialization is not possible with an
+     * invalid serial filter factory.
      * The filter factory configured using the system or security property during initialization
      * can NOT be replaced with {@link #setSerialFilterFactory(BinaryOperator) Config.setSerialFilterFactory}.
      * This ensures that a filter factory set on the command line is not overridden accidentally
@@ -582,12 +589,22 @@ final class Config {
          */
         private static volatile ObjectInputFilter serialFilter;
 
+        /**
+         * Saved message if the jdk.serialFilter property is invalid.
+         */
+        private static final String invalidFilterMessage;
+
         /**
          * Current serial filter factory.
          * @see Config#setSerialFilterFactory(BinaryOperator)
          */
         private static volatile BinaryOperator<ObjectInputFilter> serialFilterFactory;
 
+        /**
+         * Saved message if the jdk.serialFilterFactory property is invalid.
+         */
+        private static final String invalidFactoryMessage;
+
         /**
          * Boolean to indicate that the filter factory can not be set or replaced.
          * - an ObjectInputStream has already been created using the current filter factory
@@ -630,23 +647,24 @@ final class Config {
                         Security.getProperty(SERIAL_FILTER_PROPNAME));
 
             // Initialize the static filter if the jdk.serialFilter is present
-            ObjectInputFilter filter = null;
+            String filterMessage = null;
             if (filterString != null) {
                 configLog.log(DEBUG,
                         "Creating deserialization filter from {0}", filterString);
                 try {
-                    filter = createFilter(filterString);
+                    serialFilter = createFilter(filterString);
                 } catch (RuntimeException re) {
                     configLog.log(ERROR,
                             "Error configuring filter: {0}", (Object) re);
-                    // Do not continue if configuration not initialized
-                    throw re;
+                    // serialFilter remains null
+                    filterMessage = "Invalid jdk.serialFilter: " +  re.getMessage();
                 }
             }
-            serialFilter = filter;
+            invalidFilterMessage = filterMessage;
 
             // Initialize the filter factory if the jdk.serialFilterFactory is defined
             // otherwise use the builtin filter factory.
+            String factoryMessage = null;
             if (factoryClassName == null) {
                 serialFilterFactory = new BuiltinFilterFactory();
             } else {
@@ -671,10 +689,13 @@ final class Config {
                     Throwable th = (ex instanceof InvocationTargetException ite) ? ite.getCause() : ex;
                     configLog.log(ERROR,
                             "Error configuring filter factory: {0}", (Object)th);
-                    // Do not continue if configuration not initialized
-                    throw new ExceptionInInitializerError(th);
+                    // Configuration not initialized
+                    // serialFilterFactory remains null and filterFactoryNoReplace == true;
+                    factoryMessage = "invalid jdk.serialFilterFactory: " +
+                            factoryClassName + ": " + th.getClass().getName() + ": " + th.getMessage();
                 }
             }
+            invalidFactoryMessage = factoryMessage;
             // Setup shared secrets for RegistryImpl to use.
             SharedSecrets.setJavaObjectInputFilterAccess(Config::createFilter2);
         }
@@ -696,8 +717,14 @@ private static void traceFilter(String msg, Object... args) {
          * Returns the static JVM-wide deserialization filter or {@code null} if not configured.
          *
          * @return the static JVM-wide deserialization filter or {@code null} if not configured
+         * @throws IllegalStateException if the initialization of the filter from the
+         *      system property {@code jdk.serialFilter} or
+         *      the security property {@code jdk.serialFilter} fails.
          */
         public static ObjectInputFilter getSerialFilter() {
+            if (invalidFilterMessage != null) {
+                throw new IllegalStateException(invalidFilterMessage);
+            }
             return serialFilter;
         }
 
@@ -707,7 +734,9 @@ public static ObjectInputFilter getSerialFilter() {
          * @param filter the deserialization filter to set as the JVM-wide filter; not null
          * @throws SecurityException if there is security manager and the
          *       {@code SerializablePermission("serialFilter")} is not granted
-         * @throws IllegalStateException if the filter has already been set
+         * @throws IllegalStateException if the filter has already been set or the initialization
+         *       of the filter from the system property {@code jdk.serialFilter} or
+         *       the security property {@code jdk.serialFilter} fails.
          */
         public static void setSerialFilter(ObjectInputFilter filter) {
             Objects.requireNonNull(filter, "filter");
@@ -716,6 +745,9 @@ public static void setSerialFilter(ObjectInputFilter filter) {
             if (sm != null) {
                 sm.checkPermission(ObjectStreamConstants.SERIAL_FILTER_PERMISSION);
             }
+            if (invalidFilterMessage != null) {
+                throw new IllegalStateException(invalidFilterMessage);
+            }
             synchronized (serialFilterLock) {
                 if (serialFilter != null) {
                     throw new IllegalStateException("Serial filter can only be set once");
@@ -749,8 +781,10 @@ public static void setSerialFilter(ObjectInputFilter filter) {
          * @since 17
          */
         public static BinaryOperator<ObjectInputFilter> getSerialFilterFactory() {
-            if (serialFilterFactory == null)
-                throw new IllegalStateException("Serial filter factory initialization incomplete");
+            if (serialFilterFactory == null) {
+                // If initializing the factory failed or not yet complete, throw with the message
+                throw new IllegalStateException(invalidFilterFactoryMessage());
+            }
             return serialFilterFactory;
         }
 
@@ -812,15 +846,26 @@ public static void setSerialFilterFactory(BinaryOperator<ObjectInputFilter> filt
             }
             if (filterFactoryNoReplace.getAndSet(true)) {
                 final String msg = serialFilterFactory != null
-                        ? serialFilterFactory.getClass().getName()
-                        : "initialization incomplete";
-                throw new IllegalStateException("Cannot replace filter factory: " + msg);
+                        ? "Cannot replace filter factory: " + serialFilterFactory.getClass().getName()
+                        : invalidFilterFactoryMessage();
+                throw new IllegalStateException(msg);
             }
             configLog.log(DEBUG,
                     "Setting deserialization filter factory to {0}", filterFactory.getClass().getName());
             serialFilterFactory = filterFactory;
         }
 
+        /*
+         * Return message for an invalid filter factory configuration saved from the static init.
+         * It can be called before the static initializer is complete and has set the message/null.
+         */
+        private static String invalidFilterFactoryMessage() {
+            assert serialFilterFactory == null;     // undefined if a filter factory has been set
+            return (invalidFactoryMessage != null)
+                ? invalidFactoryMessage
+                : "Serial filter factory initialization incomplete";
+        }
+
         /**
          * Returns an ObjectInputFilter from a string of patterns.
          * <p>

src/java.base/share/classes/java/io/ObjectInputStream.java

@@ -384,6 +384,8 @@ private static class Logging {
      * <p>The constructor initializes the deserialization filter to the filter returned
      * by invoking the {@link Config#getSerialFilterFactory()} with {@code null} for the current filter
      * and the {@linkplain Config#getSerialFilter() static JVM-wide filter} for the requested filter.
+     * If the serial filter or serial filter factory properties are invalid
+     * an {@link IllegalStateException} is thrown.
      *
      * <p>If a security manager is installed, this constructor will check for
      * the "enableSubclassImplementation" SerializablePermission when invoked
@@ -396,6 +398,8 @@ private static class Logging {
      * @throws  IOException if an I/O error occurs while reading stream header
      * @throws  SecurityException if untrusted subclass illegally overrides
      *          security-sensitive methods
+     * @throws  IllegalStateException if the initialization of {@link ObjectInputFilter.Config}
+     *          fails due to invalid serial filter or serial filter factory properties.
      * @throws  NullPointerException if {@code in} is {@code null}
      * @see     ObjectInputStream#ObjectInputStream()
      * @see     ObjectInputStream#readFields()
@@ -421,6 +425,8 @@ public ObjectInputStream(InputStream in) throws IOException {
      * <p>The constructor initializes the deserialization filter to the filter returned
      * by invoking the {@link Config#getSerialFilterFactory()} with {@code null} for the current filter
      * and the {@linkplain Config#getSerialFilter() static JVM-wide filter} for the requested filter.
+     * If the serial filter or serial filter factory properties are invalid
+     * an {@link IllegalStateException} is thrown.
      *
      * <p>If there is a security manager installed, this method first calls the
      * security manager's {@code checkPermission} method with the
@@ -431,6 +437,8 @@ public ObjectInputStream(InputStream in) throws IOException {
      *          {@code checkPermission} method denies enabling
      *          subclassing.
      * @throws  IOException if an I/O error occurs while creating this stream
+     * @throws  IllegalStateException if the initialization of {@link ObjectInputFilter.Config}
+     *      fails due to invalid serial filter or serial filter factory properties.
      * @see SecurityManager#checkPermission
      * @see java.io.SerializablePermission
      */

test/jdk/java/io/Serializable/serialFilter/InvalidGlobalFilterTest.java

@@ -21,67 +21,100 @@
  * questions.
  */
 
-import jdk.test.lib.process.OutputAnalyzer;
-import jdk.test.lib.process.ProcessTools;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
 
-import java.io.File;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.io.ObjectInputFilter;
+import java.io.ObjectInputStream;
+import java.util.Map;
 
 /*
  * @test
- * @bug 8269336
+ * @bug 8278087
  * @summary Test that an invalid pattern value for the jdk.serialFilter system property causes an
- * exception to be thrown in the class initialization of java.io.ObjectInputFilter.Config class
- * @library /test/lib
- * @run driver InvalidGlobalFilterTest
+ * exception to be thrown when an attempt is made to use the filter or deserialize.
+ * A subset of invalid filter patterns is tested.
+ * @run testng/othervm -Djdk.serialFilter=.* InvalidGlobalFilterTest
+ * @run testng/othervm -Djdk.serialFilter=! InvalidGlobalFilterTest
+ * @run testng/othervm -Djdk.serialFilter=/ InvalidGlobalFilterTest
+ *
  */
+@Test
 public class InvalidGlobalFilterTest {
     private static final String serialPropName = "jdk.serialFilter";
+    private static final String serialFilter = System.getProperty(serialPropName);
+
+    static {
+        // Enable logging
+        System.setProperty("java.util.logging.config.file",
+                System.getProperty("test.src", ".") + "/logging.properties");
+    }
 
     /**
-     * Launches multiple instances of a Java program by passing each instance an invalid value
-     * for the {@code jdk.serialFilter} system property. The launched program then triggers the
-     * class initialization of {@code ObjectInputFilter.Config} class to have it parse the (invalid)
-     * value of the system property. The launched program is expected to propagate the exception
-     * raised by the {@code ObjectInputFilter.Config} initialization and the test asserts that the
-     * launched program did indeed fail with this expected exception.
+     * Map of invalid patterns to the expected exception message.
      */
-    public static void main(final String[] args) throws Exception {
-        final String[] invalidPatterns = {".*", ".**", "!", "/java.util.Hashtable", "java.base/", "/"};
-        for (final String invalidPattern : invalidPatterns) {
-            final ProcessBuilder processBuilder = ProcessTools.createJavaProcessBuilder(
-                    "-D" + serialPropName + "=" + invalidPattern,
-                    "-Djava.util.logging.config.file=" + System.getProperty("test.src")
-                            + File.separator + "logging.properties",
-                    ObjectInputFilterConfigLoader.class.getName());
-            // launch a process by passing it an invalid value for -Djdk.serialFilter
-            final OutputAnalyzer outputAnalyzer = ProcessTools.executeProcess(processBuilder);
-            try {
-                // we expect the JVM launch to fail
-                outputAnalyzer.shouldNotHaveExitValue(0);
-                // do an additional check to be sure it failed for the right reason
-                outputAnalyzer.stderrShouldContain("java.lang.ExceptionInInitializerError");
-            } finally {
-                // fail or pass, we print out the generated output from the launched program
-                // for any debugging
-                System.err.println("Diagnostics from process " + outputAnalyzer.pid() + ":");
-                // print out any stdout/err that was generated in the launched program
-                outputAnalyzer.reportDiagnosticSummary();
-            }
+    private static final Map<String, String> invalidMessages =
+            Map.of(".*", "Invalid jdk.serialFilter: package missing in: \".*\"",
+                    ".**", "Invalid jdk.serialFilter: package missing in: \".**\"",
+                    "!", "Invalid jdk.serialFilter: class or package missing in: \"!\"",
+                    "/java.util.Hashtable", "Invalid jdk.serialFilter: module name is missing in: \"/java.util.Hashtable\"",
+                    "java.base/", "Invalid jdk.serialFilter: class or package missing in: \"java.base/\"",
+                    "/", "Invalid jdk.serialFilter: module name is missing in: \"/\"");
+
+    @DataProvider(name = "MethodsToCall")
+    private Object[][] cases() {
+        return new Object[][] {
+                {serialFilter, "getSerialFilter", (Assert.ThrowingRunnable) () -> ObjectInputFilter.Config.getSerialFilter()},
+                {serialFilter, "setSerialFilter", (Assert.ThrowingRunnable) () -> ObjectInputFilter.Config.setSerialFilter(new NoopFilter())},
+                {serialFilter, "new ObjectInputStream(is)", (Assert.ThrowingRunnable) () -> new ObjectInputStream(new ByteArrayInputStream(new byte[0]))},
+                {serialFilter, "new OISSubclass()", (Assert.ThrowingRunnable) () -> new OISSubclass()},
+        };
+    }
+
+    /**
+     * Test each method that should throw IllegalStateException based on
+     * the invalid arguments it was launched with.
+     */
+    @Test(dataProvider = "MethodsToCall")
+    public void initFaultTest(String pattern, String method, Assert.ThrowingRunnable runnable) {
+
+        IllegalStateException ex = Assert.expectThrows(IllegalStateException.class,
+                runnable);
+
+        String expected = invalidMessages.get(serialFilter);
+        if (expected == null) {
+            Assert.fail("No expected message for filter: " + serialFilter);
         }
+        System.out.println(ex.getMessage());
+        Assert.assertEquals(ex.getMessage(), expected, "wrong message");
     }
 
-    // A main() class which just triggers the class initialization of ObjectInputFilter.Config
-    private static final class ObjectInputFilterConfigLoader {
+    private static class NoopFilter implements ObjectInputFilter {
+        /**
+         * Returns UNDECIDED.
+         *
+         * @param filter the FilterInfo
+         * @return Status.UNDECIDED
+         */
+        public ObjectInputFilter.Status checkInput(FilterInfo filter) {
+             return ObjectInputFilter.Status.UNDECIDED;
+        }
 
-        public static void main(final String[] args) throws Exception {
-            System.out.println("JVM was launched with " + serialPropName
-                    + " system property set to " + System.getProperty(serialPropName));
-            // this call is expected to fail and we aren't interested in the result.
-            // we just let the exception propagate out of this call and fail the
-            // launched program. The test which launched this main, then asserts
-            // that the exception was indeed thrown.
-            ObjectInputFilter.Config.getSerialFilter();
+        public String toString() {
+            return "NoopFilter";
         }
     }
+
+    /**
+     * Subclass of ObjectInputStream to test subclassing constructor.
+     */
+    private static class OISSubclass extends ObjectInputStream {
+
+        protected OISSubclass() throws IOException {
+        }
+    }
+
 }