0.5.0 right documentation

Author: Evgeny-Egorov-Projects

docs/cooking_machine/config_parser.html

@@ -142,6 +142,10 @@ <h1 class="title">Module <code>topicnet.cooking_machine.config_parser</code></h1
     &#34;theta_name&#34;: Str()
 }
 
+# change log style
+lc = artm.messages.ConfigureLoggingArgs()
+lc.minloglevel = 3
+lib = artm.wrapper.LibArtm(logging_config=lc)
 
 element = Any()
 base_schema = Map({

docs/cooking_machine/cubes/base_cube.html

@@ -28,7 +28,7 @@ <h1 class="title">Module <code>topicnet.cooking_machine.cubes.base_cube</code></
 from tqdm import tqdm
 import warnings
 from multiprocessing import Queue, Process
-from queue import Empty
+# from queue import Empty
 from artm.wrapper.exceptions import ArtmException
 
 from .strategy import BaseStrategy
@@ -72,14 +72,15 @@ <h1 class="title">Module <code>topicnet.cooking_machine.cubes.base_cube</code></
             return model.scores[self.score_name][-1]
 
 
-def get_from_queue_till_fail(queue,  error_message=&#39;&#39;, fail_margin=1,):
-    fail_counter = 0
-    while fail_counter &lt; fail_margin:
-        try:
-            return queue.get(timeout=0.1)
-        except Empty:
-            fail_counter += 1
-    raise Empty(error_message)
+# exists for multiprocessing debug
+def put_to_queue(queue, puttable):
+    queue.put(puttable)
+
+
+# exists for multiprocessing debug
+def get_from_queue_till_fail(queue,  error_message=&#39;&#39;,):
+    while True:
+        return queue.get()
 
 
 class BaseCube:
@@ -205,29 +206,33 @@ <h1 class="title">Module <code>topicnet.cooking_machine.cubes.base_cube</code></
                 raise ValueError(
                     f&#39;Cannot alter and fit artm model with parameters {search_point}.\n&#39;
                     &#34;ARTM failed with following: &#34; + error_message
+
                 )
             # add cube description to the model history
             new_model.add_cube(model_cube)
             new_model.experiment = experiment
             new_model.save()
+            assert os.path.exists(new_model.model_default_save_path)
 
             returned_paths.append(new_model.model_default_save_path)
 
             # some strategies depend on previous train results, therefore scores must be updated
             if self.tracked_score_function:
                 current_score = self.tracked_score_function(new_model)
-            else:
+                self.strategy.update_scores(current_score)
+            # else:
                 # we return number of iterations as a placeholder
-                current_score = len(returned_paths)
-            self.strategy.update_scores(current_score)
+                # current_score = len(returned_paths)
+
         return returned_paths
 
     def _retrieve_results_from_process(self, queue, experiment):
         from ..models import DummyTopicModel
         models_num = get_from_queue_till_fail(queue, NUM_MODELS_ERROR)
         topic_models = []
         for _ in range(models_num):
-            path = get_from_queue_till_fail(queue, MODEL_RETRIEVE_ERROR.format(_, models_num))
+            path = get_from_queue_till_fail(queue,
+                                            MODEL_RETRIEVE_ERROR.format(_, models_num))
             topic_models.append(DummyTopicModel.load(path, experiment=experiment))
 
         strategy_parameters = get_from_queue_till_fail(queue, STRATEGY_RETRIEVE_ERROR)
@@ -237,6 +242,7 @@ <h1 class="title">Module <code>topicnet.cooking_machine.cubes.base_cube</code></
         for (warning_message, warning_class) in caught_warnings:
             # if issubclass(warning_class, UserWarning):
             warnings.warn(warning_message)
+
         return topic_models
 
     def _train_models_and_report_results(self, queue, experiment, topic_model, dataset,
@@ -249,18 +255,17 @@ <h1 class="title">Module <code>topicnet.cooking_machine.cubes.base_cube</code></
         &#34;&#34;&#34;
         with warnings.catch_warnings(record=True) as caught_warnings:
             returned_paths = self._train_models(experiment, topic_model, dataset, search_space)
-
-            queue.put(len(returned_paths))
+            put_to_queue(queue, len(returned_paths))
             for path in returned_paths:
-                queue.put(path)
+                put_to_queue(queue, path)
 
             # to work with strategy we recover consistency by sending important parameters
             strategy_parameters = self.strategy._get_strategy_parameters(saveable_only=True)
-            queue.put(strategy_parameters)
+            put_to_queue(queue, strategy_parameters)
 
             caught_warnings = [(warning.message, warning.category)
                                for warning in caught_warnings]
-            queue.put(caught_warnings)
+            put_to_queue(queue, caught_warnings)
 
     def _run_cube(self, topic_model, dataset):
         &#34;&#34;&#34;
@@ -325,10 +330,10 @@ <h1 class="title">Module <code>topicnet.cooking_machine.cubes.base_cube</code></
             process = Process(
                 target=self._train_models_and_report_results,
                 args=(queue, experiment, topic_model, dataset,
-                      search_space, search_length)
+                      search_space, search_length),
+                daemon=True
             )
             process.start()
-            process.join()
             topic_models = self._retrieve_results_from_process(queue, experiment)
         else:
             returned_paths = self._train_models(experiment, topic_model, dataset, search_space)
@@ -417,22 +422,30 @@ <h2 id="returns">Returns</h2>
 </details>
 </dd>
 <dt id="topicnet.cooking_machine.cubes.base_cube.get_from_queue_till_fail"><code class="name flex">
-<span>def <span class="ident">get_from_queue_till_fail</span></span>(<span>queue, error_message='', fail_margin=1)</span>
+<span>def <span class="ident">get_from_queue_till_fail</span></span>(<span>queue, error_message='')</span>
 </code></dt>
 <dd>
 <section class="desc"></section>
 <details class="source">
 <summary>
 <span>Expand source code</span>
 </summary>
-<pre><code class="python">def get_from_queue_till_fail(queue,  error_message=&#39;&#39;, fail_margin=1,):
-    fail_counter = 0
-    while fail_counter &lt; fail_margin:
-        try:
-            return queue.get(timeout=0.1)
-        except Empty:
-            fail_counter += 1
-    raise Empty(error_message)</code></pre>
+<pre><code class="python">def get_from_queue_till_fail(queue,  error_message=&#39;&#39;,):
+    while True:
+        return queue.get()</code></pre>
+</details>
+</dd>
+<dt id="topicnet.cooking_machine.cubes.base_cube.put_to_queue"><code class="name flex">
+<span>def <span class="ident">put_to_queue</span></span>(<span>queue, puttable)</span>
+</code></dt>
+<dd>
+<section class="desc"></section>
+<details class="source">
+<summary>
+<span>Expand source code</span>
+</summary>
+<pre><code class="python">def put_to_queue(queue, puttable):
+    queue.put(puttable)</code></pre>
 </details>
 </dd>
 </dl>
@@ -594,29 +607,33 @@ <h2 id="parameters">Parameters</h2>
                 raise ValueError(
                     f&#39;Cannot alter and fit artm model with parameters {search_point}.\n&#39;
                     &#34;ARTM failed with following: &#34; + error_message
+
                 )
             # add cube description to the model history
             new_model.add_cube(model_cube)
             new_model.experiment = experiment
             new_model.save()
+            assert os.path.exists(new_model.model_default_save_path)
 
             returned_paths.append(new_model.model_default_save_path)
 
             # some strategies depend on previous train results, therefore scores must be updated
             if self.tracked_score_function:
                 current_score = self.tracked_score_function(new_model)
-            else:
+                self.strategy.update_scores(current_score)
+            # else:
                 # we return number of iterations as a placeholder
-                current_score = len(returned_paths)
-            self.strategy.update_scores(current_score)
+                # current_score = len(returned_paths)
+
         return returned_paths
 
     def _retrieve_results_from_process(self, queue, experiment):
         from ..models import DummyTopicModel
         models_num = get_from_queue_till_fail(queue, NUM_MODELS_ERROR)
         topic_models = []
         for _ in range(models_num):
-            path = get_from_queue_till_fail(queue, MODEL_RETRIEVE_ERROR.format(_, models_num))
+            path = get_from_queue_till_fail(queue,
+                                            MODEL_RETRIEVE_ERROR.format(_, models_num))
             topic_models.append(DummyTopicModel.load(path, experiment=experiment))
 
         strategy_parameters = get_from_queue_till_fail(queue, STRATEGY_RETRIEVE_ERROR)
@@ -626,6 +643,7 @@ <h2 id="parameters">Parameters</h2>
         for (warning_message, warning_class) in caught_warnings:
             # if issubclass(warning_class, UserWarning):
             warnings.warn(warning_message)
+
         return topic_models
 
     def _train_models_and_report_results(self, queue, experiment, topic_model, dataset,
@@ -638,18 +656,17 @@ <h2 id="parameters">Parameters</h2>
         &#34;&#34;&#34;
         with warnings.catch_warnings(record=True) as caught_warnings:
             returned_paths = self._train_models(experiment, topic_model, dataset, search_space)
-
-            queue.put(len(returned_paths))
+            put_to_queue(queue, len(returned_paths))
             for path in returned_paths:
-                queue.put(path)
+                put_to_queue(queue, path)
 
             # to work with strategy we recover consistency by sending important parameters
             strategy_parameters = self.strategy._get_strategy_parameters(saveable_only=True)
-            queue.put(strategy_parameters)
+            put_to_queue(queue, strategy_parameters)
 
             caught_warnings = [(warning.message, warning.category)
                                for warning in caught_warnings]
-            queue.put(caught_warnings)
+            put_to_queue(queue, caught_warnings)
 
     def _run_cube(self, topic_model, dataset):
         &#34;&#34;&#34;
@@ -714,10 +731,10 @@ <h2 id="parameters">Parameters</h2>
             process = Process(
                 target=self._train_models_and_report_results,
                 args=(queue, experiment, topic_model, dataset,
-                      search_space, search_length)
+                      search_space, search_length),
+                daemon=True
             )
             process.start()
-            process.join()
             topic_models = self._retrieve_results_from_process(queue, experiment)
         else:
             returned_paths = self._train_models(experiment, topic_model, dataset, search_space)
@@ -883,6 +900,7 @@ <h1>Index</h1>
 <ul class="">
 <li><code><a title="topicnet.cooking_machine.cubes.base_cube.check_experiment_existence" href="#topicnet.cooking_machine.cubes.base_cube.check_experiment_existence">check_experiment_existence</a></code></li>
 <li><code><a title="topicnet.cooking_machine.cubes.base_cube.get_from_queue_till_fail" href="#topicnet.cooking_machine.cubes.base_cube.get_from_queue_till_fail">get_from_queue_till_fail</a></code></li>
+<li><code><a title="topicnet.cooking_machine.cubes.base_cube.put_to_queue" href="#topicnet.cooking_machine.cubes.base_cube.put_to_queue">put_to_queue</a></code></li>
 </ul>
 </li>
 <li><h3><a href="#header-classes">Classes</a></h3>