Upgrading to 1.1.0 version for Coq v8.10.0.

Major changes in this version:
- Add a new command line interface.
- Add a new VSCode interface with a LSP server.
- In the aforementioned two interfaces, most arguments can be supplied via global or project-local .roosterizerc files.
- Drop the redundant OpenNMTInterfaceForNaming model, and simplify model selecting after only having the multi-source model.
- Add the script to package as binary distributions using PyInstaller.

Author: pengyunie

.gitignore

@@ -112,7 +112,7 @@ MANIFEST
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
-*.spec
+#*.spec
 
 # Installer logs
 pip-log.txt

Makefile

@@ -0,0 +1,25 @@
+INSTALL_DIR = $(HOME)/opt
+
+PY_SRCS = preprocess.py train.py translate.py $(wildcard roosterize/*.py) $(wildcard onmt/*.py)
+
+
+all: package
+
+.PHONY: package
+package: dist/roosterize/roosterize dist/roosterize.tgz
+
+dist/roosterize/roosterize: roosterize.spec $(PY_SRCS)
+	pyinstaller roosterize.spec -y --log-level WARN
+
+dist/roosterize.tgz: dist/roosterize/roosterize
+	cd dist && tar czf roosterize.tgz roosterize/
+
+.PHONY: install
+install: package
+	rm -rf $(INSTALL_DIR)/roosterize
+	mkdir -p $(INSTALL_DIR)/roosterize
+	cp -r dist/roosterize $(INSTALL_DIR)/roosterize/bin
+
+.PHONY: clean
+clean:
+	-rm -rf dist/ build/

onmt/utils/logging.py

@@ -1,24 +1,6 @@
-# -*- coding: utf-8 -*-
-from __future__ import absolute_import
+from seutil import LoggingUtils
 
-import logging
+logger = LoggingUtils.get_logger(__name__)
 
-logger = logging.getLogger()
-
-
-def init_logger(log_file=None, log_file_level=logging.NOTSET):
-    log_format = logging.Formatter("[%(asctime)s %(levelname)s] %(message)s")
-    logger = logging.getLogger()
-    logger.setLevel(logging.INFO)
-
-    console_handler = logging.StreamHandler()
-    console_handler.setFormatter(log_format)
-    logger.handlers = [console_handler]
-
-    if log_file and log_file != '':
-        file_handler = logging.FileHandler(log_file)
-        file_handler.setLevel(log_file_level)
-        file_handler.setFormatter(log_format)
-        logger.addHandler(file_handler)
-
-    return logger
+def init_logger(log_file=None, log_file_level=None):
+    return logger

requirements.txt

@@ -1,7 +1,10 @@
-configargparse
-nltk
+configargparse~=1.2.3
 future
-seutil==0.4.12
+nltk~=3.5
+numpy~=1.19.2
+pygls~=0.9.1
+seutil>=0.5.4
 six
-torchtext==0.4.0
-tqdm==4.30.*
+torch==1.1.0 -f https://download.pytorch.org/whl/cpu/torch_stable.html
+torchtext==0.4.0 -f https://download.pytorch.org/whl/cpu/torch_stable.html
+tqdm~=4.30.0

roosterize.spec

@@ -0,0 +1,37 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+block_cipher = None
+
+
+a = Analysis(['roosterize/main.py'],
+             pathex=['/home/pynie/projects/roosterize-vscode/python'],
+             binaries=[],
+             datas=[],
+             hiddenimports=[],
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher,
+             noarchive=False)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          [],
+          exclude_binaries=True,
+          name='roosterize',
+          debug=False,
+          bootloader_ignore_signals=False,
+          strip=False,
+          upx=True,
+          console=True )
+coll = COLLECT(exe,
+               a.binaries,
+               a.zipfiles,
+               a.datas,
+               strip=False,
+               upx=True,
+               upx_exclude=[],
+               name='roosterize')

roosterize/FilesManager.py

@@ -1,18 +1,18 @@
-from typing import *
-
 import math
-from pathlib import Path
 import traceback
-from tqdm import tqdm
+from pathlib import Path
+from typing import Any, Callable, Iterator, List, NoReturn, Optional, Union
 
 from seutil import IOUtils, LoggingUtils
+from tqdm import tqdm
+
+logger = LoggingUtils.get_logger(__name__)
 
 
 class FilesManager:
     """
     Handles the loading/dumping of files in a dataset.
     """
-    logger = LoggingUtils.get_logger(__name__)
 
     ALL_LEMMAS_BACKEND_SEXP_TRANSFORMATIONS = "all-lemmas-bsexp-transformations"
     ALL_LEMMAS_FOREEND_SEXP_TRANSFORMATIONS = "all-lemmas-fsexp-transformations"
@@ -34,7 +34,7 @@ def __init__(self, data_dir: Path):
     def clean_path(self, rel_path: Union[str, List[str]]):
         abs_path = self.data_dir / self.assemble_rel_path(rel_path)
         if abs_path.exists():
-            self.logger.info(f"Removing existing things at {abs_path}")
+            logger.info(f"Removing existing things at {abs_path}")
             IOUtils.rm(abs_path)
         # end if
         return
@@ -43,7 +43,8 @@ def clean_path(self, rel_path: Union[str, List[str]]):
     def is_json_format(cls, fmt: IOUtils.Format) -> bool:
         return fmt in [IOUtils.Format.json, IOUtils.Format.jsonPretty, IOUtils.Format.jsonNoSort]
 
-    def dump_data(self,
+    def dump_data(
+            self,
             rel_path: Union[str, List[str]],
             data: Any,
             fmt: IOUtils.Format,
@@ -53,50 +54,43 @@ def dump_data(self,
     ):
         abs_path = self.data_dir / self.assemble_rel_path(rel_path)
         if abs_path.exists() and not exist_ok:
-            LoggingUtils.log_and_raise(self.logger, f"Cannot rewrite existing data at {abs_path}", IOError)
-        # end if
+            raise IOError(f"Cannot rewrite existing data at {abs_path}")
 
         abs_path.parent.mkdir(parents=True, exist_ok=True)
         if not is_batched:
             if self.is_json_format(fmt):
                 data = IOUtils.jsonfy(data)
-            # end if
             IOUtils.dump(abs_path, data, fmt)
         else:
             # In batched mode, the data need to be slice-able and sizable
             IOUtils.rm(abs_path)
             abs_path.mkdir(parents=True)
 
-            for batch_i in tqdm(range(math.ceil(len(data)/per_batch))):
-                data_batch = data[per_batch*batch_i : per_batch*(batch_i+1)]
+            for batch_i in tqdm(range(math.ceil(len(data) / per_batch))):
+                data_batch = data[per_batch * batch_i: per_batch * (batch_i + 1)]
                 if self.is_json_format(fmt):
                     data_batch = IOUtils.jsonfy(data_batch)
-                # end if
-                IOUtils.dump(abs_path/f"batch-{batch_i}.{fmt.get_extension()}", data_batch, fmt)
-            # end for
-        # end if
+                IOUtils.dump(abs_path / f"batch-{batch_i}.{fmt.get_extension()}", data_batch, fmt)
         return
 
-    def load_data(self,
+    def load_data(
+            self,
             rel_path: Union[str, List[str]],
             fmt: IOUtils.Format,
             is_batched: bool = False,
-            clz = None,
+            clz=None,
     ) -> Any:
         if self.is_json_format(fmt) and clz is None:
-            self.logger.warning(f"Load data from {rel_path} with json format, but did not specify clz (at {traceback.format_stack()})")
-        # end if
+            logger.warning(f"Load data from {rel_path} with json format, but did not specify clz (at {traceback.format_stack()})")
 
         abs_path = self.data_dir / self.assemble_rel_path(rel_path)
         if not abs_path.exists():
-            LoggingUtils.log_and_raise(self.logger, f"Cannot find data at {abs_path}", IOError)
-        # end if
+            raise IOError(f"Cannot find data at {abs_path}")
 
         if not is_batched:
             data = IOUtils.load(abs_path, fmt)
             if self.is_json_format(fmt) and clz is not None:
                 data = IOUtils.dejsonfy(data, clz)
-            # end if
             return data
         else:
             data = list()
@@ -106,25 +100,21 @@ def load_data(self,
                 data_batch = IOUtils.load(batch_file, fmt)
                 if self.is_json_format(fmt) and clz is not None:
                     data_batch = IOUtils.dejsonfy(data_batch, clz)
-                # end if
                 data.extend(data_batch)
-            # end for
             return data
-        # end if
 
-    def iter_batched_data(self,
+    def iter_batched_data(
+            self,
             rel_path: Union[str, List[str]],
             fmt: IOUtils.Format,
-            clz = None,
+            clz=None,
     ) -> Iterator:
         if self.is_json_format(fmt) and clz is None:
-            self.logger.warning(f"Load data from {rel_path} with json format, but did not specify clz")
-        # end if
+            logger.warning(f"Load data from {rel_path} with json format, but did not specify clz")
 
         abs_path = self.data_dir / self.assemble_rel_path(rel_path)
         if not abs_path.exists():
-            LoggingUtils.log_and_raise(self.logger, f"Cannot find data at {abs_path}", IOError)
-        # end if
+            raise IOError(f"Cannot find data at {abs_path}")
 
         batch_numbers = sorted([int(str(f.stem).split("-")[1]) for f in abs_path.iterdir()])
         for batch_number in batch_numbers:
@@ -134,10 +124,12 @@ def iter_batched_data(self,
                     data_entry = IOUtils.dejsonfy(data_entry, clz)
                 # end if
                 yield data_entry
-            # end for
-        # end for
 
-    def dump_ckpt(self, rel_path: Union[str, List[str]], obj: Any, ckpt_id: int,
+    def dump_ckpt(
+            self,
+            rel_path: Union[str, List[str]],
+            obj: Any,
+            ckpt_id: int,
             dump_func: Callable[[Any, str], NoReturn],
             ckpt_keep_max: int = 5,
     ) -> NoReturn:
@@ -152,25 +144,23 @@ def dump_ckpt(self, rel_path: Union[str, List[str]], obj: Any, ckpt_id: int,
             ckpt_ids = [int(str(f.name)) for f in abs_path.iterdir()]
             for ckpt_id in sorted(ckpt_ids)[:-ckpt_keep_max]:
                 IOUtils.rm(abs_path / str(ckpt_id))
-            # end for
-        # end if
         return
 
-    def load_ckpt(self, rel_path: Union[str, List[str]],
+    def load_ckpt(
+            self,
+            rel_path: Union[str, List[str]],
             load_func: Callable[[str], Any],
             ckpt_id: Optional[int] = None,
     ) -> Any:
         abs_path = self.data_dir / self.assemble_rel_path(rel_path)
         if not abs_path.exists():
-            LoggingUtils.log_and_raise(self.logger, f"Cannot find data at {abs_path}", IOError)
-        # end if
+            raise IOError(f"Cannot find data at {abs_path}")
 
         if ckpt_id is None:
             # Find the latest ckpt
             ckpt_ids = [int(str(f.name)) for f in abs_path.iterdir()]
             ckpt_id = max(ckpt_ids)
-            self.logger.info(f"Loading the latest checkpoint {ckpt_id} at {abs_path}")
-        # end if
+            logger.info(f"Loading the latest checkpoint {ckpt_id} at {abs_path}")
 
         return load_func(str(abs_path / str(ckpt_id)))
 
@@ -181,5 +171,4 @@ def resolve(self, rel_path: Union[str, List[str]]) -> Path:
     def assemble_rel_path(cls, rel_path: Union[str, List[str]]) -> str:
         if not isinstance(rel_path, str):
             rel_path = "/".join(rel_path)
-        # end if
         return rel_path

roosterize/Macros.py

@@ -1,5 +1,3 @@
-from typing import *
-
 import os
 from pathlib import Path
 

roosterize/Utils.py

@@ -108,14 +108,3 @@ def counter_most_common_to_pretty_yaml(cls, most_common: List[Tuple[Any, int]])
         # end for
         s += "]\n"
         return s
-
-    @classmethod
-    def modify_and_import(cls, module_name, package, modification_func):
-        spec = importlib.util.find_spec(module_name, package)
-        source = spec.loader.get_source(module_name)
-        new_source = modification_func(source)
-        module = importlib.util.module_from_spec(spec)
-        codeobj = compile(new_source, module.__spec__.origin, 'exec')
-        exec(codeobj, module.__dict__)
-        sys.modules[module_name] = module
-        return module

roosterize/data/DataMiner.py

@@ -616,7 +616,8 @@ def collect_lemmas_foreend_sexp_transformations(cls, data_mgr: FilesManager):
     VTYPES_DEFINITIONS = [SexpInfo.VernacConsts.type_definition]
 
     @classmethod
-    def collect_lemmas_doc(cls,
+    def collect_lemmas_doc(
+            cls,
             doc: CoqDocument,
             ast_sexp_list: List[SexpNode],
             serapi_options: str,

roosterize/data/ModelSpec.py

@@ -26,7 +26,7 @@ def __hash__(self):
     def build_from_dict(cls, d: dict) -> "ModelSpec":
         model_spec = ModelSpec(
             name=d.get("name", ""),
-            model=d.get("model"),
+            model=d.get("model", "MultiSourceSeq2Seq"),
             config_file=d.get("config-file") if "config-file" in d else None,
             config_dict=dict(),
         )