SOLR-14067: v3 Create /contrib/scripting module with ScriptingUpdateProcessor (#2215)

* Creating Scripting contrib module to centralize the less secure code related to scripts.

* tweak the changelog and update notice to explain why the name changed and the security posture thinking

* the test script happens to be a currency.xml, which made me think we were doing something specific to currency types, but instead any xml formatted file will suffice for the test.

* drop the ing, and be more specific on the name of the ref guide page

* use the same name everywhere

Co-authored-by: David Smiley <[email protected]>

Author: epugh

gradle/maven/defaults-maven.gradle

@@ -66,6 +66,7 @@ configure(rootProject) {
         ":solr:contrib:langid",
         ":solr:contrib:jaegertracer-configurator",
         ":solr:contrib:prometheus-exporter",
+        ":solr:contrib:scripting",
         ":solr:test-framework",
     ]
 

settings.gradle

@@ -65,6 +65,7 @@ include "solr:contrib:extraction"
 include "solr:contrib:langid"
 include "solr:contrib:jaegertracer-configurator"
 include "solr:contrib:prometheus-exporter"
+include "solr:contrib:scripting"
 include "solr:contrib:ltr"
 include "solr:webapp"
 include "solr:test-framework"

solr/CHANGES.txt

@@ -188,6 +188,9 @@ Other Changes
 
 * SOLR-14297: Replace commons-codec Base64 with JDK8 Base64 (Andras Salamon via Houston Putman)
 
+* SOLR-14067: StatelessScriptUpdateProcessorFactory moved to it's own /contrib/scripting/ package instead
+ of shipping as part of Solr due to security concerns.  Renamed to ScriptUpdateProcessorFactory for simpler name. (Eric Pugh)
+ 
 Bug Fixes
 ---------------------
 * SOLR-14546: Fix for a relatively hard to hit issue in OverseerTaskProcessor that could lead to out of order execution

solr/contrib/scripting/README.md

@@ -0,0 +1,14 @@
+Welcome to Apache Solr Scripting!
+===============================
+
+# Introduction
+
+The Scripting contrib module pulls together various scripting related functions.  
+
+Today, the ScriptUpdateProcessorFactory allows Java scripting engines to support scripts written in languages such as JavaScript, Ruby, Python, and Groovy to be used during Solr document update processing, allowing dramatic flexibility in expressing custom document processing before being indexed.  It exposes hooks for commit, delete, etc, but add is the most common usage.  It is implemented as an UpdateProcessor to be placed in an UpdateChain.
+
+## Getting Started
+
+For information on how to get started please see:
+ * [Solr Reference Guide's section on Update Request Processors](https://lucene.apache.org/solr/guide/update-request-processors.html)
+  * [Solr Reference Guide's section on ScriptUpdateProcessorFactory](https://lucene.apache.org/solr/guide/script-update-processor.html)

solr/contrib/scripting/build.gradle

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply plugin: 'java-library'
+
+description = 'Scripting Package'
+
+dependencies {
+  implementation project(':solr:core')
+  testImplementation project(':solr:test-framework')
+}

solr/core/src/java/org/apache/solr/update/processor/ScriptEngineCustomizer.java renamed to solr/contrib/scripting/src/java/org/apache/solr/scripting/update/ScriptEngineCustomizer.java

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.solr.update.processor;
+package org.apache.solr.scripting.update;
 
 import javax.script.ScriptEngine;
 

solr/core/src/java/org/apache/solr/update/processor/StatelessScriptUpdateProcessorFactory.java renamed to solr/contrib/scripting/src/java/org/apache/solr/scripting/update/ScriptUpdateProcessorFactory.java

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.solr.update.processor;
+package org.apache.solr.scripting.update;
 
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
@@ -26,6 +26,8 @@
 import org.apache.solr.request.LocalSolrQueryRequest;
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.update.*;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
 import org.apache.solr.util.plugin.SolrCoreAware;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.FilenameUtils;
@@ -58,34 +60,33 @@
 
 /**
  * <p>
- * An update request processor factory that enables the use of update 
- * processors implemented as scripts which can be loaded by the 
- * {@link SolrResourceLoader} (usually via the <code>conf</code> dir for 
- * the SolrCore).
+ * An update request processor factory that enables the use of update
+ * processors implemented as scripts which can be loaded from the
+ * configSet.  Previously known as the StatelessScriptUpdateProcessorFactory.
  * </p>
  * <p>
  * This factory requires at least one configuration parameter named
- * <code>script</code> which may be the name of a script file as a string, 
- * or an array of multiple script files.  If multiple script files are 
- * specified, they are executed sequentially in the order specified in the 
+ * <code>script</code> which may be the name of a script file as a string,
+ * or an array of multiple script files.  If multiple script files are
+ * specified, they are executed sequentially in the order specified in the
  * configuration -- as if multiple factories were configured sequentially
  * </p>
  * <p>
- * Each script file is expected to declare functions with the same name 
- * as each method in {@link UpdateRequestProcessor}, using the same 
- * arguments.  One slight deviation is in the optional return value from 
- * these functions: If a script function has a <code>boolean</code> return 
- * value, and that value is <code>false</code> then the processor will 
- * cleanly terminate processing of the command and return, without forwarding 
+ * Each script file is expected to declare functions with the same name
+ * as each method in {@link UpdateRequestProcessor}, using the same
+ * arguments.  One slight deviation is in the optional return value from
+ * these functions: If a script function has a <code>boolean</code> return
+ * value, and that value is <code>false</code> then the processor will
+ * cleanly terminate processing of the command and return, without forwarding
  * the command on to the next script or processor in the chain.
- * Due to limitations in the {@link ScriptEngine} API used by 
+ * Due to limitations in the {@link ScriptEngine} API used by
  * this factory, it can not enforce that all functions exist on initialization,
  * so errors from missing functions will only be generated at runtime when
  * the chain attempts to use them.
  * </p>
  * <p>
- * The factory may also be configured with an optional "params" argument, 
- * which can be an {@link NamedList} (or array, or any other simple Java 
+ * The factory may also be configured with an optional "params" argument,
+ * which can be an {@link NamedList} (or array, or any other simple Java
  * object) which will be put into the global scope for each script.
  * </p>
  * <p>
@@ -97,40 +98,40 @@
  *  <li>params - The "params" init argument in the factory configuration (if any)</li>
  * </ul>
  * <p>
- * Internally this update processor uses JDK 6 scripting engine support, 
- * and any {@link Invocable} implementations of <code>ScriptEngine</code> 
- * that can be loaded using the Solr Plugin ClassLoader may be used.  
- * By default, the engine used for each script is determined by the filed 
- * extension (ie: a *.js file will be treated as a JavaScript script) but 
- * this can be overridden by specifying an explicit "engine" name init 
- * param for the factory, which identifies a registered name of a 
- * {@link ScriptEngineFactory}. 
- * (This may be particularly useful if multiple engines are available for 
- * the same scripting language, and you wish to force the usage of a 
+ * Internally this update processor uses JDK 6 scripting engine support,
+ * and any {@link Invocable} implementations of <code>ScriptEngine</code>
+ * that can be loaded using the Solr Plugin ClassLoader may be used.
+ * By default, the engine used for each script is determined by the file
+ * extension (ie: a *.js file will be treated as a JavaScript script) but
+ * this can be overridden by specifying an explicit "engine" name init
+ * param for the factory, which identifies a registered name of a
+ * {@link ScriptEngineFactory}.
+ * (This may be particularly useful if multiple engines are available for
+ * the same scripting language, and you wish to force the usage of a
  * particular engine because of known quirks)
  * </p>
  * <p>
- * A new {@link ScriptEngineManager} is created for each 
- * <code>SolrQueryRequest</code> defining a "global" scope for the script(s) 
- * which is request specific.  Separate <code>ScriptEngine</code> instances 
- * are then used to evaluate the script files, resulting in an "engine" scope 
+ * A new {@link ScriptEngineManager} is created for each
+ * <code>SolrQueryRequest</code> defining a "global" scope for the script(s)
+ * which is request specific.  Separate <code>ScriptEngine</code> instances
+ * are then used to evaluate the script files, resulting in an "engine" scope
  * that is specific to each script.
  * </p>
  * <p>
  * A simple example...
  * </p>
  * <pre class="prettyprint">
- * &lt;processor class="solr.StatelessScriptUpdateProcessorFactory"&gt;
+ * &lt;processor class="org.apache.solr.scripting.update.ScriptUpdateProcessorFactory"&gt;
  *   &lt;str name="script"&gt;updateProcessor.js&lt;/str&gt;
  * &lt;/processor&gt;
  * </pre>
  * <p>
- * A more complex example involving multiple scripts in different languages, 
- * and a "params" <code>NamedList</code> that will be put into the global 
+ * A more complex example involving multiple scripts in different languages,
+ * and a "params" <code>NamedList</code> that will be put into the global
  * scope of each script...
  * </p>
  * <pre class="prettyprint">
- * &lt;processor class="solr.StatelessScriptUpdateProcessorFactory"&gt;
+ * &lt;processor class="org.apache.solr.scripting.update.ScriptUpdateProcessorFactory"&gt;
  *   &lt;arr name="script"&gt;
  *     &lt;str name="script"&gt;first-processor.js&lt;/str&gt;
  *     &lt;str name="script"&gt;second-processor.py&lt;/str&gt;
@@ -142,22 +143,22 @@
  * &lt;/processor&gt;
  * </pre>
  * <p>
- * An example where the script file extensions are ignored, and an 
+ * An example where the script file extensions are ignored, and an
  * explicit script engine is used....
  * </p>
  * <pre class="prettyprint">
- * &lt;processor class="solr.StatelessScriptUpdateProcessorFactory"&gt;
+ * &lt;processor class="org.apache.solr.scripting.update.ScriptUpdateProcessorFactory"&gt;
  *   &lt;arr name="script"&gt;
  *     &lt;str name="script"&gt;first-processor.txt&lt;/str&gt;
  *     &lt;str name="script"&gt;second-processor.txt&lt;/str&gt;
  *   &lt;/arr&gt;
  *   &lt;str name="engine"&gt;rhino&lt;/str&gt;
  * &lt;/processor&gt;
  * </pre>
- * 
+ *
  * @since 4.0.0
  */
-public class StatelessScriptUpdateProcessorFactory extends UpdateRequestProcessorFactory implements SolrCoreAware {
+public class ScriptUpdateProcessorFactory extends UpdateRequestProcessorFactory implements SolrCoreAware {
 
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
@@ -182,8 +183,8 @@ public void init(@SuppressWarnings({"rawtypes"})NamedList args) {
     Collection<String> scripts =
       args.removeConfigArgs(SCRIPT_ARG);
     if (scripts.isEmpty()) {
-      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, 
-                              "StatelessScriptUpdateProcessorFactory must be " +
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+                              "ScriptUpdateProcessorFactory must be " +
                               "initialized with at least one " + SCRIPT_ARG);
     }
     scriptFiles = new ArrayList<>();
@@ -199,8 +200,8 @@ public void init(@SuppressWarnings({"rawtypes"})NamedList args) {
         engineName = (String)engine;
       } else {
         throw new SolrException
-          (SolrException.ErrorCode.SERVER_ERROR, 
-           "'" + ENGINE_NAME_ARG + "' init param must be a String (found: " + 
+          (SolrException.ErrorCode.SERVER_ERROR,
+           "'" + ENGINE_NAME_ARG + "' init param must be a String (found: " +
            engine.getClass() + ")");
       }
     }
@@ -246,7 +247,7 @@ public void inform(SolrCore core) {
       req.close();
     }
 
-    
+
   }
 
 
@@ -259,13 +260,13 @@ public void inform(SolrCore core) {
    * @param rsp The solr response
    * @return The list of initialized script engines.
    */
-  private List<EngineInfo> initEngines(SolrQueryRequest req, 
-                                       SolrQueryResponse rsp) 
+  private List<EngineInfo> initEngines(SolrQueryRequest req,
+                                       SolrQueryResponse rsp)
     throws SolrException {
-    
+
     List<EngineInfo> scriptEngines = new ArrayList<>();
 
-    ScriptEngineManager scriptEngineManager 
+    ScriptEngineManager scriptEngineManager
       = new ScriptEngineManager(resourceLoader.getClassLoader());
 
     scriptEngineManager.put("logger", log);
@@ -281,29 +282,29 @@ private List<EngineInfo> initEngines(SolrQueryRequest req,
         engine = scriptEngineManager.getEngineByName(engineName);
         if (engine == null) {
           String details = getSupportedEngines(scriptEngineManager, false);
-          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, 
+          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
                                   "No ScriptEngine found by name: "
-                                  + engineName + 
-                                  (null != details ? 
+                                  + engineName +
+                                  (null != details ?
                                    " -- supported names: " + details : ""));
         }
       } else {
         engine = scriptEngineManager.getEngineByExtension
           (scriptFile.getExtension());
         if (engine == null) {
           String details = getSupportedEngines(scriptEngineManager, true);
-          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, 
+          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
                                   "No ScriptEngine found by file extension: "
-                                  + scriptFile.getFileName() + 
-                                  (null != details ? 
+                                  + scriptFile.getFileName() +
+                                  (null != details ?
                                    " -- supported extensions: " + details : ""));
-                                  
+
         }
       }
 
       if (! (engine instanceof Invocable)) {
-        String msg = 
-          "Engine " + ((null != engineName) ? engineName : 
+        String msg =
+          "Engine " + ((null != engineName) ? engineName :
                        ("for script " + scriptFile.getFileName())) +
           " does not support function invocation (via Invocable): " +
           engine.getClass().toString() + " (" +
@@ -319,7 +320,7 @@ private List<EngineInfo> initEngines(SolrQueryRequest req,
       scriptEngines.add(new EngineInfo((Invocable)engine, scriptFile));
       try {
         Reader scriptSrc = scriptFile.openReader(resourceLoader);
-  
+
         try {
           try {
             AccessController.doPrivileged(new PrivilegedExceptionAction<Void>() {
@@ -333,23 +334,23 @@ public Void run() throws ScriptException  {
             throw (ScriptException) e.getException();
           }
         } catch (ScriptException e) {
-          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, 
-                                  "Unable to evaluate script: " + 
+          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+                                  "Unable to evaluate script: " +
                                   scriptFile.getFileName(), e);
         } finally {
           IOUtils.closeQuietly(scriptSrc);
         }
       } catch (IOException ioe) {
-        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, 
-            "Unable to evaluate script: " + 
-            scriptFile.getFileName(), ioe);        
+        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+            "Unable to evaluate script: " +
+            scriptFile.getFileName(), ioe);
       }
     }
     return scriptEngines;
   }
 
   /**
-   * For error messages - returns null if there are any exceptions of any 
+   * For error messages - returns null if there are any exceptions of any
    * kind building the string (or of the list is empty for some unknown reason).
    * @param ext - if true, list of extensions, otherwise a list of engine names
    */
@@ -403,7 +404,7 @@ public void processDelete(DeleteUpdateCommand cmd) throws IOException {
       if (invokeFunction("processDelete", cmd)) {
         super.processDelete(cmd);
       }
-        
+
     }
 
     @Override
@@ -435,9 +436,9 @@ public void finish() throws IOException {
     }
 
     /**
-     * returns true if processing should continue, or false if the 
-     * request should be ended now.  Result value is computed from the return 
-     * value of the script function if: it exists, is non-null, and can be 
+     * returns true if processing should continue, or false if the
+     * request should be ended now.  Result value is computed from the return
+     * value of the script function if: it exists, is non-null, and can be
      * cast to a java Boolean.
      */
     private boolean invokeFunction(String name, Object... cmd) {
@@ -461,10 +462,10 @@ private boolean invokeFunctionUnsafe(String name, Object... cmd) {
           }
 
         } catch (ScriptException | NoSuchMethodException e) {
-          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, 
-                                  "Unable to invoke function " + name + 
-                                  " in script: " + 
-                                  engine.getScriptFile().getFileName() + 
+          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+                                  "Unable to invoke function " + name +
+                                  " in script: " +
+                                  engine.getScriptFile().getFileName() +
                                   ": " + e.getMessage(), e);
         }
       }