Make CRF and elastic setup optional

* downgrade elasticsearch-dsl version due to compatibility issues
* make elastic connection setup optional
* make CRF loading optional
* disable multiprocessing in tokenization to avoid forking in celery
workers

Author: mkardas

environment.yml

@@ -13,7 +13,7 @@ dependencies:
 - python=3.7.1
 - pyahocorasick=1.4.0
 - Unidecode=1.0.23
-- elasticsearch-dsl=7.0.0
+- elasticsearch-dsl=6.3.1
 - ipython=7.5.0
 - joblib=0.13.2
 - python-magic=0.4.15

sota_extractor2/data/structure.py

@@ -191,9 +191,10 @@ def prepare_data(tables, csv_path, cache=False):
 
 
 class CellEvidenceExtractor:
-    def __init__(self):
+    def __init__(self, setup_connection=True):
         # todo: make sure can be called more than once or refactor to singleton
-        setup_default_connection()
+        if setup_connection:
+            setup_default_connection()
 
     def __call__(self, paper, tables, paper_limit=30, corpus_limit=10):
         dfs = [evidence_for_table(paper.paper_id, table, paper_limit, corpus_limit) for table in tables]

sota_extractor2/models/structure/structure_predictor.py

@@ -35,17 +35,19 @@ def cut_ulmfit_head(model):
 class TableStructurePredictor(ULMFiT_SP):
     step = "structure_prediction"
 
-    def __init__(self, path, file, crf_path=None, crf_model="crf.pkl",
+    def __init__(self, path, file, crf_path=None, crf_model=None,
                  sp_path=None, sp_model="spm.model", sp_vocab="spm.vocab"):
         super().__init__(path, file, sp_path, sp_model, sp_vocab)
 
         self._full_learner = deepcopy(self.learner)
         self.learner.model = cut_ulmfit_head(self.learner.model)
         self.learner.loss_func = None
 
-        #todo: make CRF optional
-        crf_path = Path(path) if crf_path is None else Path(crf_path)
-        self.crf = load_crf(crf_path / crf_model)
+        if crf_model is not None:
+            crf_path = Path(path) if crf_path is None else Path(crf_path)
+            self.crf = load_crf(crf_path / crf_model)
+        else:
+            self.crf = None
 
         # todo: clean Experiment from older approaches
         self._e = ULMFiTExperiment(remove_num=False, drop_duplicates=False,

sota_extractor2/models/structure/ulmfit.py

@@ -13,6 +13,7 @@ def _fix_sp_processor(self, sp_path, sp_model, sp_vocab):
             if isinstance(processor, SPProcessor):
                 processor.sp_model = sp_path / sp_model
                 processor.sp_vocab = sp_path / sp_vocab
+                processor.n_cpus = 1
 
                 #todo: see why it wasn't set on save
                 processor.mark_fields = True