Merge pull request #9 from 3D-e-Chem/knime-43

Support KNIME 4.3

Author: sverhoeven

CHANGELOG.md

@@ -5,6 +5,16 @@ The file is formatted as described on http://keepachangelog.com/.
 
 ## [Unreleased]
 
+## [2.0.4] - 2020-01-14
+
+### Fixed
+
+* Support KNIME 4.3 ([#9](https://github.com/3D-e-Chem/knime-python-wrapper/pull/9) + [#10](https://github.com/3D-e-Chem/knime-python-wrapper/pull/10))
+
+### Removed
+
+* Support for KNIME versions older than 4.3
+
 ## [2.0.3] - 2019-07-02
 
 ### Changed
@@ -53,3 +63,11 @@ The file is formatted as described on http://keepachangelog.com/.
 
 * Abstract PythonWrapperNode classes
 * Test utility to run tests which call PythonKernel execute
+
+[Unreleased]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v2.0.4...HEAD
+[2.0.4]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v2.0.3...v2.0.4
+[2.0.3]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v2.0.2...v2.0.3
+[2.0.2]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v2.0.1...v2.0.2
+[2.0.1]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v1.1.0...v2.0.1
+[1.1.0]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v1.0.0...v1.1.0
+[1.0.0]: https://github.com/3D-e-Chem/knime-python-wrapper/releases/tag/v1.0.0

p2/category.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <site>
-   <bundle id="nl.esciencecenter.e3dchem.python.plugin" version="2.0.3.qualifier">
+   <bundle id="nl.esciencecenter.e3dchem.python.plugin" version="2.0.4.qualifier">
       <category name="nl.esciencecenter.3D-e-Chem"/>
    </bundle>
    <category-def name="nl.esciencecenter.3D-e-Chem" label="KNIME 3D-e-Chem Contributions">

p2/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>nl.esciencecenter.e3dchem.python</artifactId>
         <groupId>nl.esciencecenter.e3dchem.python</groupId>
-        <version>2.0.3-SNAPSHOT</version>
+        <version>2.0.4-SNAPSHOT</version>
     </parent>
     <artifactId>nl.esciencecenter.e3dchem.python.p2</artifactId>
     <packaging>eclipse-repository</packaging>

plugin/META-INF/MANIFEST.MF

@@ -2,15 +2,15 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: Abstract Python wrapper KNIME node and helpers
 Bundle-SymbolicName: nl.esciencecenter.e3dchem.python.plugin;singleton:=true
-Bundle-Version: 2.0.3.qualifier
+Bundle-Version: 2.0.4.qualifier
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8
 Bundle-Vendor: Netherlands eScience Center
-Require-Bundle: org.knime.core;bundle-version="[4.0.0,5.0.0)",
- org.knime.base;bundle-version="[4.0.0,5.0.0)",
- org.knime.python2;bundle-version="[4.0.0,5.0.0)",
- org.knime.python2.serde.csv;bundle-version="[4.0.0,5.0.0)",
- org.knime.python2.serde.flatbuffers;bundle-version="[4.0.0,5.0.0)",
- org.knime.python2.serde.arrow;bundle-version="[4.0.0,5.0.0)",
- org.knime.python2.serde.arrow.libs;bundle-version="[4.0.0,5.0.0)"
+Require-Bundle: org.knime.core;bundle-version="[4.3.0,5.0.0)",
+ org.knime.base;bundle-version="[4.3.0,5.0.0)",
+ org.knime.python2;bundle-version="[4.3.0,5.0.0)",
+ org.knime.python2.envconfigs;bundle-version="[4.3.0,5.0.0)",
+ org.knime.python2.serde.csv;bundle-version="[4.3.0,5.0.0)",
+ org.knime.python2.serde.flatbuffers;bundle-version="[4.3.0,5.0.0)",
+ org.knime.python2.serde.arrow;bundle-version="[4.3.0,5.0.0)"
 Bundle-ClassPath: .
 Export-Package: nl.esciencecenter.e3dchem.python

plugin/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>nl.esciencecenter.e3dchem.python</groupId>
 		<artifactId>nl.esciencecenter.e3dchem.python</artifactId>
-		<version>2.0.3-SNAPSHOT</version>
+		<version>2.0.4-SNAPSHOT</version>
 	</parent>
 	<artifactId>nl.esciencecenter.e3dchem.python.plugin</artifactId>
 	<packaging>eclipse-plugin</packaging>

plugin/src/java/nl/esciencecenter/e3dchem/python/PythonWrapperNodeConfig.java

@@ -1,26 +1,22 @@
 package nl.esciencecenter.e3dchem.python;
 
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Locale;
-import java.util.Objects;
 import java.util.Set;
 
-import org.knime.python2.DefaultPythonCommand;
+import org.knime.core.node.InvalidSettingsException;
+import org.knime.core.node.NodeSettingsRO;
+import org.knime.core.node.NodeSettingsWO;
+import org.knime.core.node.workflow.FlowVariable;
 import org.knime.python2.PythonCommand;
 import org.knime.python2.PythonModuleSpec;
 import org.knime.python2.PythonVersion;
+import org.knime.python2.config.PythonCommandFlowVariableConfig;
 import org.knime.python2.extensions.serializationlibrary.SentinelOption;
 import org.knime.python2.extensions.serializationlibrary.SerializationOptions;
 import org.knime.python2.generic.VariableNames;
 import org.knime.python2.kernel.PythonKernelOptions;
 import org.knime.python2.prefs.PythonPreferences;
-import org.knime.core.node.InvalidSettingsException;
-import org.knime.core.node.NodeSettingsRO;
-import org.knime.core.node.NodeSettingsWO;
-import org.knime.core.node.workflow.FlowVariable;
 
 /**
  * Configuration for {@link PythonWrapperNodeModel}.
@@ -31,32 +27,22 @@
  */
 public class PythonWrapperNodeConfig {
 	private static final String CFG_PYTHON_VERSION_OPTION = "pythonVersionOption";
-    static final String CFG_PYTHON2COMMAND = "python2Command";
-    static final String CFG_PYTHON3COMMAND = "python3Command";
+    private static final String CFG_PYTHON2_COMMAND = "python2Command";
+
+    private static final String CFG_PYTHON3_COMMAND = "python3Command";
 	private static final String CFG_CONVERT_MISSING_TO_PYTHON = "convertMissingToPython";
 	private static final String CFG_CONVERT_MISSING_FROM_PYTHON = "convertMissingFromPython";
 	private static final String CFG_SENTINEL_OPTION = "sentinelOption";
 	private static final String CFG_SENTINEL_VALUE = "sentinelValue";
 	private static final String CFG_CHUNK_SIZE = "chunkSize";
 
-    /**
-     * {@link #m_python2Command} and {@link #m_python3Command} are special in that they currently aren't configurable
-     * via a Python scripting node's dialog but only using flow variables. If no respective flow variables are set,
-     * their value is retrieved from the Python preferences.
-     */
-    private static final String INDICATE_FALLBACK_TO_PREFERENCES_COMMAND_VALUE = "";
-
     private PythonVersion m_pythonVersion = PythonPreferences.getPythonVersionPreference();
 
-    /**
-     * {@code null} means to fall back to {@link PythonPreferences#getPython2CommandPreference()}.
-     */
-    private PythonCommand m_python2Command = null;
+    private PythonCommandFlowVariableConfig m_python2CommandConfig = new PythonCommandFlowVariableConfig(CFG_PYTHON2_COMMAND,
+        PythonVersion.PYTHON2, PythonPreferences::getCondaInstallationPath);
 
-    /**
-     * {@code null} means to fall back to {@link PythonPreferences#getPython2CommandPreference()}.
-     */
-    private PythonCommand m_python3Command = null;
+    private PythonCommandFlowVariableConfig m_python3CommandConfig = new PythonCommandFlowVariableConfig(CFG_PYTHON3_COMMAND,
+        PythonVersion.PYTHON3, PythonPreferences::getCondaInstallationPath);
 
     private int m_chunkSize = SerializationOptions.DEFAULT_CHUNK_SIZE;
 
@@ -118,13 +104,20 @@ public void setPythonVersion(final PythonVersion pythonVersion) {
         m_pythonVersion = pythonVersion;
     }
 
+    /**
+     * @return The config of the Python 2 command.
+     */
+    public PythonCommandFlowVariableConfig getPython2CommandConfig() {
+        return m_python2CommandConfig;
+    }
+
     /**
      * @return The Python 2 command to use. May be {@code null} in which case no specific Python 2 command is configured
      *         and one has to resort to - e.g., - the {@link PythonPreferences#getPython2CommandPreference() global
      *         preferences}.
      */
     public PythonCommand getPython2Command() {
-        return m_python2Command;
+        return m_python2CommandConfig.getCommand().orElse(null);
     }
 
     /**
@@ -133,16 +126,23 @@ public PythonCommand getPython2Command() {
      *            {@link PythonPreferences#getPython2CommandPreference() global preferences}.
      */
     public void setPython2Command(final PythonCommand python2Command) {
-        m_python2Command = python2Command;
+        m_python2CommandConfig.setCommand(python2Command);
+    }
+
+    /**
+     * @return The config of the Python 3 command.
+     */
+    public PythonCommandFlowVariableConfig getPython3CommandConfig() {
+        return m_python3CommandConfig;
     }
 
     /**
      * @return The Python 3 command to use. May be {@code null} in which case no specific Python 3 command is configured
      *         and one has to resort to - e.g., - the {@link PythonPreferences#getPython3CommandPreference() global
      *         preferences}.
      */
-    public PythonCommand getPython3Command() {
-        return m_python3Command;
+	public PythonCommand getPython3Command() {
+        return m_python3CommandConfig.getCommand().orElse(null);
     }
 
     /**
@@ -151,9 +151,9 @@ public PythonCommand getPython3Command() {
      *            {@link PythonPreferences#getPython3CommandPreference() global preferences}.
      */
     public void setPython3Command(final PythonCommand python3Command) {
-        m_python3Command = python3Command;
+        m_python3CommandConfig.setCommand(python3Command);
     }
-
+    
     /**
     *
     * @return The configured number of rows to transfer to/from Python per chunk of an input/output table.
@@ -273,13 +273,13 @@ public void saveTo(final NodeSettingsWO settings) {
 	 */
 	public void saveToInDialog(NodeSettingsWO settings) {
         settings.addString(CFG_PYTHON_VERSION_OPTION, getPythonVersion().getId());
-        settings.addString(CFG_PYTHON2COMMAND, commandToString(getPython2Command()));
-        settings.addString(CFG_PYTHON3COMMAND, commandToString(getPython3Command()));
         settings.addInt(CFG_CHUNK_SIZE, getChunkSize());
         settings.addBoolean(CFG_CONVERT_MISSING_TO_PYTHON, getConvertMissingToPython());
         settings.addBoolean(CFG_CONVERT_MISSING_FROM_PYTHON, getConvertMissingFromPython());
         settings.addString(CFG_SENTINEL_OPTION, getSentinelOption().name());
         settings.addInt(CFG_SENTINEL_VALUE, getSentinelValue());
+        m_python2CommandConfig.saveSettingsTo(settings);
+        m_python3CommandConfig.saveSettingsTo(settings);
    	}
 
 	/**
@@ -300,39 +300,22 @@ public void loadFrom(final NodeSettingsRO settings) throws InvalidSettingsExcept
 	 *
 	 * @param settings
 	 *            The settings to load from
+	 * @throws InvalidSettingsException 
 	 */
-	public void loadFromInDialog(final NodeSettingsRO settings) {
+	public void loadFromInDialog(final NodeSettingsRO settings) throws InvalidSettingsException {
 		final String pythonVersionString = settings.getString(CFG_PYTHON_VERSION_OPTION, getPythonVersion().getId());
         // Backward compatibility: old saved versions may be all upper case.
         setPythonVersion(PythonVersion.fromId(pythonVersionString.toLowerCase(Locale.ROOT)));
-        final String python2CommandString =
-            settings.getString(CFG_PYTHON2COMMAND, commandToString(getPython2Command()));
-        setPython2Command(commandFromString(python2CommandString));
-        final String python3CommandString =
-            settings.getString(CFG_PYTHON3COMMAND, commandToString(getPython3Command()));
-        setPython3Command(commandFromString(python3CommandString));
         setChunkSize(settings.getInt(CFG_CHUNK_SIZE, getChunkSize()));
         setConvertMissingToPython(settings.getBoolean(CFG_CONVERT_MISSING_TO_PYTHON, getConvertMissingToPython()));
         setConvertMissingFromPython(
             settings.getBoolean(CFG_CONVERT_MISSING_FROM_PYTHON, getConvertMissingFromPython()));
         setSentinelOption(SentinelOption.valueOf(settings.getString(CFG_SENTINEL_OPTION, getSentinelOption().name())));
         setSentinelValue(settings.getInt(CFG_SENTINEL_VALUE, getSentinelValue()));
+        m_python2CommandConfig.loadSettingsFrom(settings);
+        m_python3CommandConfig.loadSettingsFrom(settings);        
 	}
 
-    private static String commandToString(final PythonCommand command) {
-        return command != null //
-            ? command.toString() //
-            : INDICATE_FALLBACK_TO_PREFERENCES_COMMAND_VALUE;
-    }
-
-    private static PythonCommand commandFromString(final String commandString) {
-        return Objects.equals(commandString, INDICATE_FALLBACK_TO_PREFERENCES_COMMAND_VALUE) //
-            ? null //
-            // TODO: This only works for ordinary paths ("manual configuration"), not for Conda directory + environment
-            // name ("Conda configuration").
-            : new DefaultPythonCommand(commandString);
-    }
-    
 	/**
 	 * Set of key/value pairs which inside Python script will be a dictionary
 	 * named by {@link #getOptionsName()}.
@@ -352,9 +335,8 @@ public Set<FlowVariable> getOptionsValues() {
     public PythonKernelOptions getKernelOptions() {
         final SerializationOptions serializationOptions = new SerializationOptions(getChunkSize(),
             getConvertMissingToPython(), getConvertMissingFromPython(), getSentinelOption(), getSentinelValue());
-		PythonKernelOptions opt = new PythonKernelOptions(getPythonVersion(), getPython2Command(), getPython3Command(),
-            serializationOptions);
-		return opt.forAddedAdditionalRequiredModules(additionalRequiredModules).forExternalCustomPath(externalCustomPath);
+    	PythonKernelOptions opt = new PythonKernelOptions();
+		return opt.forSerializationOptions(serializationOptions).forAddedAdditionalRequiredModules(additionalRequiredModules).forExternalCustomPath(externalCustomPath);
     }
 
 	/**

plugin/src/java/nl/esciencecenter/e3dchem/python/PythonWrapperNodeModel.java

@@ -7,8 +7,10 @@
 import java.io.InputStreamReader;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.knime.base.node.util.exttool.ExtToolOutputNodeModel;
@@ -22,14 +24,21 @@
 import org.knime.core.node.port.PortType;
 import org.knime.core.node.workflow.FlowVariable;
 import org.knime.core.node.workflow.FlowVariable.Type;
+import org.knime.python2.PythonCommand;
+import org.knime.python2.PythonModuleSpec;
+import org.knime.python2.config.PythonFlowVariableOptions;
+import org.knime.python2.kernel.PythonCancelable;
+import org.knime.python2.kernel.PythonCanceledExecutionException;
 import org.knime.python2.kernel.PythonExecutionMonitorCancelable;
+import org.knime.python2.kernel.PythonIOException;
 import org.knime.python2.kernel.PythonKernel;
+import org.knime.python2.kernel.PythonKernelOptions;
+import org.knime.python2.kernel.PythonKernelQueue;
 
 /**
  * Implements a {@link NodeModel} for nodes that launch external Python script.
  *
- * @param <C>
- *            Configuration
+ * @param <C> Configuration
  */
 public abstract class PythonWrapperNodeModel<C extends PythonWrapperNodeConfig> extends ExtToolOutputNodeModel {
 	protected C m_config = createConfig();
@@ -48,14 +57,52 @@ protected final C getConfig() {
 		return m_config;
 	}
 
+	/**
+	 * Gets the kernel specific options.
+	 *
+	 * @return the kernel specific options
+	 */
+	protected PythonKernelOptions getKernelOptions() {
+		final PythonKernelOptions options = getConfig().getKernelOptions();
+		final String serializerId = new PythonFlowVariableOptions(getAvailableFlowVariables()).getSerializerId()
+				.orElse(null);
+		return options.forSerializationOptions(options.getSerializationOptions().forSerializerId(serializerId));
+	}
+
+	// Below has been copied from KNIME Python node code (https://github.com/knime/knime-python/blob/analytics-platform/4.3.0/org.knime.python2.nodes/src/org/knime/python2/nodes/PythonNodeModel.java) and
+	// adjusted.
+	protected PythonKernel getNextKernelFromQueue(final PythonCancelable cancelable)
+			throws PythonCanceledExecutionException, PythonIOException {
+		return getNextKernelFromQueue(Collections.emptySet(), Collections.emptySet(), cancelable);
+	}
+
+	protected PythonKernel getNextKernelFromQueue(final Set<PythonModuleSpec> requiredAdditionalModules,
+			final PythonCancelable cancelable) throws PythonCanceledExecutionException, PythonIOException {
+		return getNextKernelFromQueue(requiredAdditionalModules, Collections.emptySet(), cancelable);
+	}
+
+	protected PythonKernel getNextKernelFromQueue(final Set<PythonModuleSpec> requiredAdditionalModules,
+			final Set<PythonModuleSpec> optionalAdditionalModules, final PythonCancelable cancelable)
+			throws PythonCanceledExecutionException, PythonIOException {
+		final PythonKernelOptions options = getKernelOptions();
+		final PythonCommand command = options.getUsePython3() //
+				? options.getPython3Command() //
+				: options.getPython2Command();
+		return PythonKernelQueue.getNextKernel(command, requiredAdditionalModules, optionalAdditionalModules, options,
+				cancelable);
+	}
+
 	public BufferedDataTable[] execute(BufferedDataTable[] inData, ExecutionContext exec) throws Exception {
-		// Below has been copied from Knime Python node source code and
+		// Below has been copied from KNIME Python node source code (https://github.com/knime/knime-python/blob/analytics-platform/4.3.0/org.knime.python2.nodes/src/org/knime/python2/nodes/source/PythonSourceNodeModel.java) and
 		// adjusted.
-		PythonKernel kernel = new PythonKernel(getConfig().getKernelOptions());
-		try {
-			return executeKernel(inData, exec, kernel);
-		} finally {
-			kernel.close();
+		final PythonExecutionMonitorCancelable cancelable = new PythonExecutionMonitorCancelable(exec);
+		try (final PythonKernel kernel = getNextKernelFromQueue(cancelable)) {
+			kernel.setOptions(getConfig().getKernelOptions());
+			try {
+				return executeKernel(inData, exec, kernel);
+			} finally {
+				kernel.close();
+			}
 		}
 	}
 
@@ -96,11 +143,10 @@ public BufferedDataTable[] executeKernel(BufferedDataTable[] inData, ExecutionCo
 	/**
 	 * Push new variables to the stack.
 	 *
-	 * Only pushes new variables to the stack if they are new or changed in type
-	 * or value.
+	 * Only pushes new variables to the stack if they are new or changed in type or
+	 * value.
 	 *
-	 * @param newVariables
-	 *            The flow variables to push
+	 * @param newVariables The flow variables to push
 	 */
 	protected void addNewVariables(Collection<FlowVariable> newVariables) {
 		// Below has been copied from Knime Python node source code and