Merge branch 'master' into readme-update

Author: pengyunie

bin/roosterize

@@ -4,5 +4,5 @@ _DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 
 
 ( cd $_DIR/../
-  python -m roosterize.main $@
+  python -m roosterize.main "$@"
 )

dist-bin/roosterize

@@ -4,5 +4,5 @@ _DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 
 
 ( cd $_DIR/../
-  ./roosterize $@
+  ./roosterize "$@"
 )

pretrain.sh

@@ -9,27 +9,23 @@ _DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 function process_data() {
         local ds=$1; shift
 
-        ( cd $_DIR/python/
-          rm -rf $_DIR/output/$ds/data
-          python -m roosterize.main extract_data_from_corpus\
-                 --corpus=$_DIR/../math-comp-corpus\
-                 --output=$_DIR/output/$ds/data\
-                 --groups=$ds
-        )
+        rm -rf $_DIR/output/$ds/data
+        python -m roosterize.main extract_data_from_corpus\
+               --corpus=$_DIR/../math-comp-corpus\
+               --output=$_DIR/output/$ds/data\
+               --groups=$ds
 }
 
 function train_model() {
         local ds=$1; shift
 
-        ( cd $_DIR/python/
-          rm -rf $_DIR/output/$ds/model
-          python -m roosterize.main train_model\
-                 --train=$_DIR/output/$ds/data/$ds-train\
-                 --val=$_DIR/output/$ds/data/$ds-val\
-                 --model-dir=$_DIR/output/$ds/model\
-                 --output=$_DIR/output/$ds/data\
-                 --config-file=$_DIR/configs/Stmt+ChopKnlTree+attn+copy.json
-        )
+        rm -rf $_DIR/output/$ds/model
+        python -m roosterize.main train_model\
+               --train=$_DIR/output/$ds/data/$ds-train\
+               --val=$_DIR/output/$ds/data/$ds-val\
+               --model-dir=$_DIR/output/$ds/model\
+               --output=$_DIR/output/$ds/data\
+               --config-file=$_DIR/configs/Stmt+ChopKnlTree+attn+copy.json
 }
 
 function package_model() {
@@ -43,13 +39,11 @@ function package_model() {
 function eval_model() {
         local ds=$1; shift
 
-        ( cd $_DIR/python/
-          rm -rf $_DIR/output/$ds/results
-          python -m roosterize.main eval_model\
-                 --data=$_DIR/output/$ds/data/$ds-test\
-                 --model-dir=$_DIR/output/$ds/model\
-                 --output=$_DIR/output/$ds/results
-        )
+        rm -rf $_DIR/output/$ds/results
+        python -m roosterize.main eval_model\
+               --data=$_DIR/output/$ds/data/$ds-test\
+               --model-dir=$_DIR/output/$ds/model\
+               --output=$_DIR/output/$ds/results
 }
 
 function retrain_all_models() {

roosterize/ml/naming/MultiSourceSeq2Seq.py

@@ -349,7 +349,9 @@ def process_data_impl(
         # Inputs
         all_inputs: Dict[str, List[List[str]]] = self.get_all_inputs(lemmas, docs_sub_tokenizers)
         for input_type, src_sentences in all_inputs.items():
-            IOUtils.dump(output_processed_data_dir/f"src.{input_type}.txt", src_sentences, IOUtils.Format.txtList)
+            IOUtils.dump(output_processed_data_dir/f"src.{input_type}.txt",
+                "".join([" ".join(sent) + "\n" for sent in src_sentences]),
+                IOUtils.Format.txt)
 
         # Outputs
         IOUtils.dump(