Add methods to read/write multiple-document YAML files (#1085)

Author: rakillen

core/src/main/java/oracle/weblogic/deploy/yaml/AbstractYamlTranslator.java

@@ -39,55 +39,84 @@ public abstract class AbstractYamlTranslator {
 
     protected abstract PlatformLogger getLogger();
     protected abstract String getClassName();
-    public abstract PyDictionary parse() throws YamlException;
+
+    // override to return a list of documents as Python dictionaries from the YAML
+    public abstract PyList parseDocuments(boolean allowMultiple) throws YamlException;
+
+    // override to write a list of documents as Python dictionaries to the YAML
+    public abstract void dumpDocuments(List<?> documents) throws YamlException;
 
     protected AbstractYamlTranslator(String fileName, boolean useOrderedDict) {
         this.fileName = fileName;
         this.useOrderedDict = useOrderedDict;
     }
 
+    /**
+     * This method triggers parsing of the file and conversion into a Python dictionary.
+     * If the YAML is empty, a single empty dictionary is returned.
+     * If the YAML contains multiple documents, a YamlException is thrown
+     *
+     * @return the python dictionary corresponding to the YAML input file
+     * @throws YamlException if an error occurs while reading the input file
+     */
+    public PyDictionary parse() throws YamlException {
+        final String METHOD = "parse";
+        PyDictionary result;
+        PyList dictionaries = parseDocuments(false);
+        if (dictionaries.isEmpty()) {
+            result = getNewDictionary();
+        } else {
+            result = (PyDictionary) dictionaries.get(0);
+        }
+        return result;
+    }
+
     @SuppressWarnings("WeakerAccess")
-    protected PyDictionary parseInternal(InputStream inputStream) throws YamlException {
+    protected PyList parseInternal(InputStream inputStream, boolean allowMultiple) throws YamlException {
         final String METHOD = "parseInternal";
 
-        PyDictionary result = null;
+        // there are problems using PyList.add(),
+        // so build a java.util.List and construct PyList(javaList).
+        List<PyObject> result = new ArrayList<>();
         if (inputStream != null) {
             Yaml parser = new Yaml(this.getDefaultLoaderOptions());
 
-            PyDictionary firstDoc = null;
-            int docCount = 0;
             try {
                 Iterable<Object> docsIterable = parser.loadAll(inputStream);
-
-                Object docToConvert = null;
+                List<Object> documents = new ArrayList<>();
                 for (Object document : docsIterable) {
-                    docCount++;
-                    if (docCount == 1) {
-                        docToConvert = document;
-                    }
+                    documents.add(document);
                 }
-                if (docCount == 1) {
-                    firstDoc = convertJavaDataStructureToPython(docToConvert);
-                } else if (docCount == 0) {
-                    firstDoc = getNewDictionary();
+
+                // don't continue with conversion if multiple documents check fails
+                if(!allowMultiple && documents.size() > 1) {
+                    YamlException pex = new YamlException("WLSDPLY-18101", this.fileName, documents.size());
+                    getLogger().throwing(getClassName(), METHOD, pex);
+                    throw pex;
                 }
+
+                for (Object document : documents) {
+                    result.add(convertJavaDataStructureToPython(document));
+                }
+            } catch (YamlException yex) {
+                throw yex;
             } catch (Exception ex) {
                 YamlException pex = new YamlException("WLSDPLY-18100", ex, this.fileName, ex.getLocalizedMessage());
                 getLogger().throwing(getClassName(), METHOD, pex);
                 throw pex;
             }
-            if (docCount > 1) {
-                YamlException pex = new YamlException("WLSDPLY-18101", this.fileName, docCount);
-                getLogger().throwing(getClassName(), METHOD, pex);
-                throw pex;
-            }
-            result = firstDoc;
         }
-        return result;
+        return new PyList(result.toArray(new PyObject[0]));
+    }
+
+    public void dump(Map<String, Object> data) throws YamlException {
+        List<Map<String, Object>> list = new ArrayList<>();
+        list.add(data);
+        dumpDocuments(list);
     }
 
     @SuppressWarnings("WeakerAccess")
-    protected void dumpInternal(Map<String, Object> data, Writer outputWriter) throws YamlException {
+    protected void dumpInternal(List<?> data, Writer outputWriter) throws YamlException {
         final String METHOD = "dumpInternal";
 
         if (outputWriter != null) {
@@ -96,7 +125,7 @@ protected void dumpInternal(Map<String, Object> data, Writer outputWriter) throw
             Yaml yaml = new Yaml(representer, dumperOptions);
 
             try {
-                yaml.dump(replaceNoneInMap(data), outputWriter);
+                yaml.dumpAll(data.iterator(), outputWriter);
             } catch (Exception ex) {
                 YamlException pex = new YamlException("WLSDPLY-18107", ex, this.fileName, ex.getLocalizedMessage());
                 getLogger().throwing(getClassName(), METHOD, pex);

core/src/main/java/oracle/weblogic/deploy/yaml/YamlStreamTranslator.java

@@ -7,12 +7,12 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.Writer;
-import java.util.Map;
+import java.util.List;
 
 import oracle.weblogic.deploy.logging.PlatformLogger;
 import oracle.weblogic.deploy.logging.WLSDeployLogFactory;
 
-import org.python.core.PyDictionary;
+import org.python.core.PyList;
 
 /**
  * An implementation of the YAML parser/translator that reads the YAML input from an input stream.
@@ -63,21 +63,21 @@ public YamlStreamTranslator(String streamFileName, Writer yamlOutputWriter) {
     }
 
     /**
-     * This method triggers parsing of the YAML and conversion into the Python dictionary.  Note that is closes
-     * the input stream when it is finished, making the instance no longer viable.
+     * Read a list of documents as Python dictionaries from the YAML input stream.
+     * Note that the input stream is closed when it is finished, making the instance no longer viable.
      *
-     * @return the python dictionary corresponding to the YAML input
+     * @return a list of Python dictionaries corresponding to the YAML input
      * @throws YamlException if an error occurs while reading the input
      */
     @Override
-    public PyDictionary parse() throws YamlException {
-        final String METHOD = "parse";
+    public PyList parseDocuments(boolean allowMultiple) throws YamlException {
+        final String METHOD = "parseDocuments";
 
         LOGGER.entering(CLASS, METHOD);
-        PyDictionary result = null;
+        PyList result = null;
         if (yamlStream != null) {
             try {
-                result = parseInternal(yamlStream);
+                result = parseInternal(yamlStream, allowMultiple);
             } finally {
                 try {
                     yamlStream.close();
@@ -91,8 +91,8 @@ public PyDictionary parse() throws YamlException {
         return result;
     }
 
-    public void dump(Map<String, Object> data) throws YamlException {
-        final String METHOD = "dump";
+    public void dumpDocuments(List<?> data) throws YamlException {
+        final String METHOD = "dumpDocuments";
 
         LOGGER.entering(CLASS, METHOD);
         if (yamlOutputWriter != null) {

core/src/main/java/oracle/weblogic/deploy/yaml/YamlTranslator.java

@@ -8,13 +8,13 @@
 import java.io.FileInputStream;
 import java.io.FileWriter;
 import java.io.IOException;
-import java.util.Map;
+import java.util.List;
 
 import oracle.weblogic.deploy.logging.PlatformLogger;
 import oracle.weblogic.deploy.logging.WLSDeployLogFactory;
 import oracle.weblogic.deploy.util.FileUtils;
 
-import org.python.core.PyDictionary;
+import org.python.core.PyList;
 
 /**
  * An implementation of the YAML parser/translator that reads the YAML input from an input stream.
@@ -46,20 +46,21 @@ public YamlTranslator(String fileName, boolean useOrderedDict) {
         super(fileName, useOrderedDict);
         this.yamlFile = FileUtils.validateExistingFile(fileName);
     }
+
     /**
-     * This method triggers parsing of the file and conversion into the Python dictionary.
+     * Read a list of documents as Python dictionaries from the YAML file.
      *
-     * @return the python dictionary corresponding to the YAML input file
-     * @throws YamlException if an error occurs while reading the input file
+     * @return a list of documents corresponding to the YAML input
+     * @throws YamlException if an error occurs while reading the input
      */
     @Override
-    public PyDictionary parse() throws YamlException {
-        final String METHOD = "parse";
+    public PyList parseDocuments(boolean allowMultiple) throws YamlException {
+        final String METHOD = "parseDocuments";
 
         LOGGER.entering(CLASS, METHOD);
-        PyDictionary result;
+        PyList result;
         try (FileInputStream fis = new FileInputStream(yamlFile)) {
-            result = parseInternal(fis);
+            result = parseInternal(fis, allowMultiple);
         } catch (IOException ioe) {
             YamlException ex = new YamlException("WLSDPLY-18108", ioe, yamlFile.getPath(), ioe.getLocalizedMessage());
             LOGGER.throwing(CLASS, METHOD, ex);
@@ -71,8 +72,8 @@ public PyDictionary parse() throws YamlException {
         return result;
     }
 
-    public void dump(Map<String, Object> data) throws YamlException {
-        final String METHOD = "dump";
+    public void dumpDocuments(List<?> data) throws YamlException {
+        final String METHOD = "dumpDocuments";
 
         // Don't log the data since it is big and could contain credentials.
         LOGGER.entering(CLASS, METHOD);

core/src/main/python/compare_model.py

@@ -209,7 +209,7 @@ def compare(self):
             print format_message('WLSDPLY-05707')
             print BLANK_LINE
             pty = PythonToYaml(change_model)
-            pty._write_dictionary_to_yaml_file(change_model, System.out)
+            pty.write_to_stream(System.out)
 
         return 0
 

core/src/main/python/wlsdeploy/yaml/yaml_translator.py

@@ -14,6 +14,7 @@
 import java.lang.Long as JLong
 import java.lang.String as JString
 import java.util.ArrayList as JArrayList
+from java.io import OutputStreamWriter
 
 import oracle.weblogic.deploy.util.FileUtils as JFileUtils
 import oracle.weblogic.deploy.yaml.YamlStreamTranslator as JYamlStreamTranslator
@@ -63,6 +64,22 @@ def parse(self):
         self._logger.exiting(class_name=self._class_name, method_name=_method_name)
         return result_dict
 
+    def parse_documents(self):
+        """
+        Parse the Yaml content from the file and convert it to a list of Python dictionaries.
+        :return: a list of documents as Python dictionaries
+        :raises: YamlException: if an error occurs while parsing the Yaml or converting it to the list
+        """
+        _method_name = 'parse_documents'
+
+        self._logger.entering(class_name=self._class_name, method_name=_method_name)
+        # throws YamlException with details, nothing we can really add here...
+        result_dict = self._translator.parseDocuments(True)
+
+        # don't log the model on exit, it may contain passwords
+        self._logger.exiting(class_name=self._class_name, method_name=_method_name)
+        return result_dict
+
 
 class YamlStreamToPython(object):
     """
@@ -102,21 +119,24 @@ def parse(self):
 
 class PythonToJava(object):
     """
-    A class that converts a Python dictionary and its contents to the Java types expected by snakeyaml.
+    A class that converts a Python dictionary or document list and its contents
+    to the Java types expected by snakeyaml.
     """
     _class_name = 'PythonToJava'
 
-    def __init__(self, dictionary):
-        self._dictionary = dictionary
+    def __init__(self, collection):
+        self._collection = collection
         self._logger = PlatformLogger('wlsdeploy.yaml')
 
     def convert_to_java(self):
         _method_name = 'convert_to_java'
 
-        if self._dictionary is None:
+        if self._collection is None:
             return None
-        if isinstance(self._dictionary, dict):
-            return self.convert_dict_to_java_map(self._dictionary)
+        if isinstance(self._collection, dict):
+            return self.convert_dict_to_java_map(self._collection)
+        if isinstance(self._collection, list):
+            return self.convert_list_to_java_list(self._collection)
         else:
             yaml_ex = exception_helper.create_yaml_exception('WLSDPLY-18200')
             self._logger.throwing(class_name=self._class_name, method_name=_method_name, error=yaml_ex)
@@ -181,13 +201,13 @@ def convert_scalar_to_java_type(self, py_value):
 
 class PythonToYaml(object):
     """
-    A class that converts a Python dictionary into Yaml and writes the output to a file.
+    A class that converts a Python dictionary or document list into Yaml and writes the output to a file.
     """
     _class_name = 'PythonToYaml'
 
-    def __init__(self, dictionary):
+    def __init__(self, collection):
         # Fix error handling for None
-        self._dictionary = dictionary
+        self._collection = collection
         self._logger = PlatformLogger('wlsdeploy.yaml')
         return
 
@@ -212,7 +232,7 @@ def write_to_yaml_file(self, file_name):
         writer = None
         try:
             writer = JFileWriter(yaml_file)
-            self._write_dictionary_to_yaml_file(self._dictionary, writer, file_name)
+            self._write_collection_to_yaml_file(self._collection, writer, file_name)
         except JFileNotFoundException, fnfe:
             yaml_ex = exception_helper.create_yaml_exception('WLSDPLY-18010', file_name,
                                                              fnfe.getLocalizedMessage(), error=fnfe)
@@ -228,20 +248,27 @@ def write_to_yaml_file(self, file_name):
 
         self._logger.exiting(class_name=self._class_name, method_name=_method_name)
 
-    def _write_dictionary_to_yaml_file(self, dictionary, writer, file_name='<None>'):
+    def write_to_stream(self, output_stream):
+        self._write_collection_to_yaml_file(self._collection, OutputStreamWriter(output_stream))
+
+    def _write_collection_to_yaml_file(self, collection, writer, file_name='<None>'):
         """
-        Do the actual heavy lifting of converting a dictionary and writing it to the file.
-        :param dictionary: the Python dictionary to convert
+        Do the actual heavy lifting of converting a dictionary or document list and writing it to the file.
+        :param collection: the Python dictionary or document list to convert
         :param writer: the java.io.Writer for the output file
         :param file_name: the file_name for the output file
         :raises: YamlException: if an error occurs while writing the output
         """
-        if dictionary is None:
+        if collection is None:
             return
 
-        java_object = PythonToJava(dictionary).convert_to_java()
+        java_object = PythonToJava(collection).convert_to_java()
         yaml_stream_translator = JYamlStreamTranslator(file_name, writer)
-        yaml_stream_translator.dump(java_object)
+
+        if isinstance(collection, list):
+            yaml_stream_translator.dumpDocuments(java_object)
+        else:
+            yaml_stream_translator.dump(java_object)
         return
 
     def _close_writer(self, writer):