Make tests more concise/consistent

Fixtures are used consistently
A new TempFolder class has been created for reusability
A session-wide trained_model fixture has been added to speed up testing

Author: MatthewScholefield

test/scripts/conftest.py

@@ -12,21 +12,51 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import shutil
+
 import pytest
 
 from precise.scripts.train import TrainScript
-from test.scripts.test_train import DummyTrainFolder
+from test.scripts.test_utils.temp_folder import TempFolder
+from test.scripts.test_utils.dummy_train_folder import DummyTrainFolder
 
 
 @pytest.fixture()
 def train_folder():
-    folder = DummyTrainFolder(10)
+    folder = DummyTrainFolder()
+    folder.generate_default()
     try:
         yield folder
     finally:
         folder.cleanup()
 
 
 @pytest.fixture()
-def train_script(train_folder):
-    return TrainScript.create(model=train_folder.model, folder=train_folder.root, epochs=1)
+def temp_folder():
+    folder = TempFolder()
+    try:
+        yield folder
+    finally:
+        folder.cleanup()
+
+
+@pytest.fixture(scope='session')
+def _trained_model():
+    """Session wide model that gets trained once"""
+    folder = DummyTrainFolder()
+    folder.generate_default()
+    script = TrainScript.create(model=folder.model, folder=folder.root, epochs=100)
+    script.run()
+    try:
+        yield folder.model
+    finally:
+        folder.cleanup()
+
+
+@pytest.fixture()
+def trained_model(_trained_model, temp_folder):
+    """Copy of session wide model"""
+    model = temp_folder.path('trained_model.net')
+    shutil.copy(_trained_model, model)
+    shutil.copy(_trained_model + '.params', model + '.params')
+    return model

test/scripts/test_add_noise.py

@@ -13,25 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from precise.scripts.add_noise import AddNoiseScript
-
-from test.scripts.dummy_audio_folder import DummyAudioFolder
-
-
-class DummyNoiseFolder(DummyAudioFolder):
-    def __init__(self, count=10):
-        super().__init__(count)
-        self.source = self.subdir('source')
-        self.noise = self.subdir('noise')
-        self.output = self.subdir('output')
-
-        self.generate_samples(self.subdir('source', 'wake-word'), 'ww-{}.wav', 1.0, self.rand(0, 2))
-        self.generate_samples(self.subdir('source', 'not-wake-word'), 'nww-{}.wav', 0.0, self.rand(0, 2))
-        self.generate_samples(self.noise, 'noise-{}.wav', 0.5, self.rand(10, 20))
+from test.scripts.test_utils.dummy_noise_folder import DummyNoiseFolder
 
 
 class TestAddNoise:
     def get_base_data(self, count):
-        folders = DummyNoiseFolder(count)
+        folders = DummyNoiseFolder()
+        folders.generate_default(count)
         base_args = dict(
             folder=folders.source, noise_folder=folders.noise,
             output_folder=folders.output
@@ -42,10 +30,10 @@ def test_run_basic(self):
         folders, base_args = self.get_base_data(10)
         script = AddNoiseScript.create(inflation_factor=1, **base_args)
         script.run()
-        assert folders.count_files(folders.output) == 20
+        assert folders.count_files(folders.output) == 40
 
     def test_run_basic_2(self):
         folders, base_args = self.get_base_data(10)
         script = AddNoiseScript.create(inflation_factor=2, **base_args)
         script.run()
-        assert folders.count_files(folders.output) == 40
+        assert folders.count_files(folders.output) == 80

test/scripts/test_combined.py

@@ -18,35 +18,35 @@
 from precise.scripts.calc_threshold import CalcThresholdScript
 from precise.scripts.eval import EvalScript
 from precise.scripts.graph import GraphScript
+from test.scripts.test_utils.dummy_train_folder import DummyTrainFolder
 
 
 def read_content(filename):
     with open(filename) as f:
         return f.read()
 
 
-def test_combined(train_folder, train_script):
+def test_combined(train_folder: DummyTrainFolder, trained_model: str):
     """Test a "normal" development cycle, train, evaluate and calc threshold.
     """
-    train_script.run()
-    params_file = train_folder.model + '.params'
-    assert isfile(train_folder.model)
+    params_file = trained_model + '.params'
+    assert isfile(trained_model)
     assert isfile(params_file)
 
     EvalScript.create(folder=train_folder.root,
-                      models=[train_folder.model]).run()
+                      models=[trained_model]).run()
 
     # Ensure that the graph script generates a numpy savez file
     out_file = train_folder.path('outputs.npz')
     graph_script = GraphScript.create(folder=train_folder.root,
-                                      models=[train_folder.model],
+                                      models=[trained_model],
                                       output_file=out_file)
     graph_script.run()
     assert isfile(out_file)
 
     # Esure the params are updated after threshold is calculated
     params_before = read_content(params_file)
     CalcThresholdScript.create(folder=train_folder.root,
-                               model=train_folder.model,
+                               model=trained_model,
                                input_file=out_file).run()
     assert params_before != read_content(params_file)

test/scripts/test_convert.py

@@ -15,10 +15,10 @@
 from os.path import isfile
 
 from precise.scripts.convert import ConvertScript
+from test.scripts.test_utils.temp_folder import TempFolder
 
 
-def test_convert(train_folder, train_script):
-    train_script.run()
-
-    ConvertScript.create(model=train_folder.model, out=train_folder.model + '.pb').run()
-    assert isfile(train_folder.model + '.pb')
+def test_convert(temp_folder: TempFolder, trained_model: str):
+    pb_model = temp_folder.path('model.pb')
+    ConvertScript.create(model=trained_model, out=pb_model).run()
+    assert isfile(pb_model)

test/scripts/test_engine.py

@@ -21,6 +21,8 @@
 from precise.scripts.engine import EngineScript
 from runner.precise_runner import ReadWriteStream
 
+from test.scripts.test_utils.dummy_train_folder import DummyTrainFolder
+
 
 class FakeStdin:
     def __init__(self, data: bytes):
@@ -35,18 +37,17 @@ def __init__(self):
         self.buffer = ReadWriteStream()
 
 
-def test_engine(train_folder, train_script):
+def test_engine(train_folder: DummyTrainFolder, trained_model: str):
     """
-    Test t hat the output format of the engina matches a decimal form in the
+    Test t hat the output format of the engine matches a decimal form in the
     range 0.0 - 1.0.
     """
-    train_script.run()
     with open(glob.glob(join(train_folder.root, 'wake-word', '*.wav'))[0], 'rb') as f:
         data = f.read()
     try:
         sys.stdin = FakeStdin(data)
         sys.stdout = FakeStdout()
-        EngineScript.create(model_name=train_folder.model).run()
+        EngineScript.create(model_name=trained_model).run()
         assert re.match(rb'[01]\.[0-9]+', sys.stdout.buffer.buffer)
     finally:
         sys.stdin = sys.__stdin__

test/scripts/test_train.py

@@ -14,29 +14,14 @@
 # limitations under the License.
 from os.path import isfile
 
-from precise.params import pr
 from precise.scripts.train import TrainScript
-from test.scripts.dummy_audio_folder import DummyAudioFolder
-
-
-class DummyTrainFolder(DummyAudioFolder):
-    def __init__(self, count=10):
-        super().__init__(count)
-        self.generate_samples(self.subdir('wake-word'), 'ww-{}.wav', 1.0,
-                              self.rand(0, 2 * pr.buffer_t))
-        self.generate_samples(self.subdir('not-wake-word'), 'nww-{}.wav', 0.0,
-                              self.rand(0, 2 * pr.buffer_t))
-        self.generate_samples(self.subdir('test', 'wake-word'), 'ww-{}.wav',
-                              1.0, self.rand(0, 2 * pr.buffer_t))
-        self.generate_samples(self.subdir('test', 'not-wake-word'),
-                              'nww-{}.wav', 0.0, self.rand(0, 2 * pr.buffer_t))
-        self.model = self.path('model.net')
+from test.scripts.test_utils.dummy_train_folder import DummyTrainFolder
 
 
 class TestTrain:
-    def test_run_basic(self):
+    def test_run_basic(self, train_folder: DummyTrainFolder):
         """Run a training and check that a model is generated."""
-        folders = DummyTrainFolder(10)
-        script = TrainScript.create(model=folders.model, folder=folders.root)
-        script.run()
-        assert isfile(folders.model)
+        train_script = TrainScript.create(model=train_folder.model, folder=train_folder.root, epochs=10)
+        train_script.run()
+        assert isfile(train_folder.model)
+        assert isfile(train_folder.model + '.params')

test/scripts/test_utils/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2020 Mycroft AI Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

test/scripts/test_utils/dummy_noise_folder.py

@@ -0,0 +1,35 @@
+# Copyright 2020 Mycroft AI Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import numpy as np
+
+from test.scripts.test_utils.temp_folder import TempFolder
+from test.scripts.test_utils.dummy_train_folder import DummyTrainFolder
+
+
+class DummyNoiseFolder(TempFolder):
+    def __init__(self):
+        super().__init__()
+        self.source = self.subdir('source')
+        self.noise = self.subdir('noise')
+        self.output = self.subdir('output')
+
+        self.source_folder = DummyTrainFolder(root=self.source)
+        self.noise_folder = DummyTrainFolder(root=self.noise)
+
+    def generate_default(self, count=10):
+        self.source_folder.generate_default(count)
+        self.noise_folder.generate_samples(
+            count, [], 'noise-{}.wav',
+            lambda: np.ones(self.noise_folder.get_duration(), dtype=float)
+        )

test/scripts/test_utils/dummy_train_folder.py

@@ -0,0 +1,68 @@
+# Copyright 2020 Mycroft AI Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import random
+from os.path import join
+
+import numpy as np
+
+from precise.params import pr
+from precise.util import save_audio
+from test.scripts.test_utils.temp_folder import TempFolder
+
+
+class DummyTrainFolder(TempFolder):
+    def __init__(self, root=None):
+        super().__init__(root)
+        self.model = self.path('model.net')
+
+    def generate_samples(self, count, subfolder, name, generator):
+        """
+        Generate sample audio files in a folder
+
+        The file is generated in the specified folder, with the specified
+        name and generated value.
+
+        Args:
+            count: Number of samples to generate
+            subfolder: String or list of subfolder path
+            name: Format string used to generate each sample
+            generator: Function called to get the data for each sample
+        """
+        if isinstance(subfolder, str):
+            subfolder = [subfolder]
+        for i in range(count):
+            save_audio(join(self.subdir(*subfolder), name.format(i)), generator())
+
+    def get_duration(self):
+        """Generate a random sample duration"""
+        return int(random.random() * 2 * pr.buffer_samples)
+
+    def generate_default(self, count=10):
+        self.generate_samples(
+            count, 'wake-word', 'ww-{}.wav',
+            lambda: np.ones(self.get_duration(), dtype=float)
+        )
+        self.generate_samples(
+            count, 'not-wake-word', 'nww-{}.wav',
+            lambda: np.random.random(self.get_duration()) * 2 - 1
+        )
+        self.generate_samples(
+            count, ('test', 'wake-word'), 'ww-{}.wav',
+            lambda: np.ones(self.get_duration(), dtype=float)
+        )
+        self.generate_samples(
+            count, ('test', 'not-wake-word'), 'nww-{}.wav',
+            lambda: np.random.random(self.get_duration()) * 2 - 1
+        )
+        self.model = self.path('model.net')

test/scripts/dummy_audio_folder.py test/scripts/test_utils/temp_folder.pytest/scripts/dummy_audio_folder.py renamed to test/scripts/test_utils/temp_folder.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright 2019 Mycroft AI Inc.
+# Copyright 2020 Mycroft AI Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,36 +14,18 @@
 # limitations under the License.
 import atexit
 
-import numpy as np
 import os
 from os import makedirs
 from os.path import isdir, join
 from shutil import rmtree
 from tempfile import mkdtemp
 
-from precise.params import pr
-from precise.util import save_audio
 
-
-class DummyAudioFolder:
-    def __init__(self, count=10):
-        self.count = count
-        self.root = mkdtemp()
+class TempFolder:
+    def __init__(self, root=None):
+        self.root = mkdtemp() if root is None else root
         atexit.register(self.cleanup)
 
-    def rand(self, min, max):
-        return min + (max - min) * np.random.random() * pr.buffer_t
-
-    def generate_samples(self, folder, name, value, duration):
-        """Generate sample file.
-
-        The file is generated in the specified folder, with the specified name,
-        dummy value and duration.
-        """
-        for i in range(self.count):
-            save_audio(join(folder, name.format(i)),
-                       np.array([value] * int(duration * pr.sample_rate)))
-
     def subdir(self, *parts):
         folder = self.path(*parts)
         if not isdir(folder):