Merge branch 'master' of github.com:udapi/udapi-python

Author: dan-zeman

udapi/block/read/sentences.py

@@ -56,3 +56,8 @@ def read_tree(self, document=None):
         if self.newdoc_if_empty_line and preceded_by_empty_line:
             root.newdoc = True
         return root
+
+    # The first line in a file also marks a start of new document
+    def after_process_document(self, document):
+        if self.newdoc_if_empty_line:
+            document.bundles[0].trees[0].newdoc = True

udapi/block/util/wc.py

@@ -23,7 +23,7 @@ def process_tree(self, tree):
         self.mwts += mwtoks
         self.tokens += len(tree.token_descendants) if mwtoks else len(tree.descendants)
         self.empty += len(tree.empty_nodes)
-        if tree.newdoc:
+        if tree.newdoc or tree == tree.document[0].trees[0]:
             self.docs += 1
         if tree.newpar:
             self.paragraphs += 1

udapi/core/basereader.py

@@ -314,6 +314,6 @@ def read_documents(self):
         docs = []
         while not self.finished:
             doc = Document()
-            self.process_document(doc)
+            self.apply_on_document(doc)
             docs.append(doc)
         return docs

udapi/core/basewriter.py

@@ -2,6 +2,7 @@
 import sys
 import logging
 import os
+from pathlib import Path
 
 import udapi.core.coref
 from udapi.core.block import Block
@@ -62,7 +63,7 @@ def before_process_document(self, document):
                 docname = document.meta.get('docname', None)
                 if docname is not None:
                     logging.info('Writing to file %s.', docname)
-                    sys.stdout = open(docname, 'wt', encoding=self.encoding, newline=self.newline)
+                    sys.stdout = self._open(docname)
                 else:
                     logging.warning('docname_as_file=1 but the document contains no docname')
             elif self.overwrite or self.path:
@@ -81,7 +82,7 @@ def before_process_document(self, document):
                             new_ext = old_ext
                         docname = os.path.join(new_dir, new_file + new_ext)
                     logging.info('Writing to file %s.', docname)
-                    sys.stdout = open(docname, 'wt', encoding=self.encoding, newline=self.newline)
+                    sys.stdout = self._open(docname)
                 else:
                     logging.warning('using overwrite or path but document.meta["loaded_from"] is None')
             else:
@@ -96,10 +97,13 @@ def before_process_document(self, document):
                 sys.stdout = self.orig_stdout
             else:
                 logging.info('Writing to file %s.', filename)
-                sys.stdout = open(filename, 'wt', encoding=self.encoding, newline=self.newline)
+                sys.stdout = self._open(filename)
         if old_filehandle not in (sys.stdout, self.orig_stdout):
             old_filehandle.close()
 
+    def _open(self, filename):
+        Path(filename).parent.mkdir(parents=True, exist_ok=True)
+        return open(filename, 'wt', encoding=self.encoding, newline=self.newline)
 
     def after_process_document(self, document):
         sys.stdout.flush()

udapi/core/root.py

@@ -178,6 +178,10 @@ def create_multiword_token(self, words=None, form=None, misc=None):
         """
         mwt = MWT(words, form, misc, root=self)
         self._mwts.append(mwt)
+        if words[-1].misc["SpaceAfter"] == "No":
+            mwt.misc["SpaceAfter"] = "No"
+        for word in words:
+            word.misc["SpaceAfter"] = ""
         return mwt
 
     @property

udapi/core/run.py

@@ -80,9 +80,13 @@ def _blocks_in_a_package(package_name):
             pname = pname[12:]
         blocks = []
         for sname in submodule_names:
-            module =  __import__(f"{package_name}.{sname}", fromlist="dummy")
-            bname = [c for c in dir(module) if c.lower() == sname][0]
-            blocks.append(f"{pname}.{bname}")
+            try: # ignore modules with compilation errors
+                module =  __import__(f"{package_name}.{sname}", fromlist="dummy")
+                bnames = [c for c in dir(module) if c.lower() == sname]
+                if bnames:
+                    blocks.append(f"{pname}.{bnames[0]}")
+            except:
+                pass
         return blocks
     except:
             return []
@@ -114,13 +118,13 @@ def _import_blocks(block_names, block_args):
             exec(command)  # pylint: disable=exec-used
         except ModuleNotFoundError as err:
             package_name = ".".join(module.split(".")[:-1])
-            blocks = _blocks_in_a_package(package_name)
-            if not blocks:
+            package_blocks = _blocks_in_a_package(package_name)
+            if not package_blocks:
                 raise
             raise ModuleNotFoundError(
                 f"Cannot find block {block_name} (i.e. class {module}.{class_name})\n"
                 f"Available block in {package_name} are:\n"
-                + "\n".join(_blocks_in_a_package(package_name))) from err
+                + "\n".join(package_blocks)) from err
         except Exception as ex:
             logging.warning(f"Cannot import block {block_name} (i.e. class {module}.{class_name})")
             raise