Gpu param for command line, tests.

Author: Arkkienkeli

deepprofiler/__main__.py

@@ -22,7 +22,7 @@
               type=click.Path("r"))
 @click.option("--config", default=None,
               help="Path to existing config file",
-              type=click.Path("r"))
+              type=click.STRING)
 @click.option("--cores", default=0,
               help="Number of CPU cores for parallel processing (all=0)",
               type=click.INT)
@@ -52,13 +52,11 @@ def cli(context, root, config, exp, cores, gpu):
         "summaries": root+"/outputs/" + exp + "/summaries/",
         "features": root+"/outputs/" + exp + "/features/"
     }
-    if config is not None:
-        context.obj["config"] = {}
-        context.obj["config"]["paths"] = {}
-        context.obj["config"]["paths"]["config"] = config
-        dirs["config"] = os.path.dirname(os.path.abspath(config))
-    else:
-        config = dirs["config"] + "/config.json"
+
+    context.obj["config"] = {}
+    context.obj["config"]["paths"] = {}
+    context.obj["config"]["paths"]["config"] = root+"/inputs/config/" + config
+
     context.obj["cores"] = cores
     context.obj["gpu"] = gpu
 

plugins/models/autoencoder.py

@@ -66,6 +66,6 @@ def define_model(config, dset):
 
 
 class ModelClass(DeepProfilerModel):
-    def __init__(self, config, dset, generator, val_generator):
-        super(ModelClass, self).__init__(config, dset, generator, val_generator)
+    def __init__(self, config, dset, gpu, generator, val_generator):
+        super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator)
         self.feature_model, self.encoder, self.decoder, self.optimizer, self.loss = define_model(config, dset)

plugins/models/cnn.py

@@ -52,6 +52,6 @@ def define_model(config, dset):
 
 
 class ModelClass(DeepProfilerModel):
-    def __init__(self, config, dset, generator, val_generator):
-        super(ModelClass, self).__init__(config, dset, generator, val_generator)
+    def __init__(self, config, dset, gpu, generator, val_generator):
+        super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator)
         self.feature_model, self.optimizer, self.loss = define_model(config, dset)

plugins/models/densenet.py

@@ -47,7 +47,7 @@ def define_model(config, dset):
     return model, optimizer, loss_func
 
 class ModelClass(DeepProfilerModel):
-    def __init__(self, config, dset, generator, val_generator):
-        super(ModelClass, self).__init__(config, dset, generator, val_generator)
+    def __init__(self, config, dset, gpu, generator, val_generator):
+        super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator)
         self.feature_model, self.optimizer, self.loss = define_model(config, dset)
 

plugins/models/gan.py

@@ -140,8 +140,8 @@ def train(self, epochs, steps_per_epoch, init_epoch):
 
 
 class ModelClass(DeepProfilerModel):
-    def __init__(self, config, dset, generator, val_generator):
-        super(ModelClass, self).__init__(config, dset, generator, val_generator)
+    def __init__(self, config, dset, gpu, generator, val_generator):
+        super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator)
         self.gan = GAN(config, self.train_crop_generator, self.val_crop_generator)
         self.feature_model = self.gan.discriminator
 

plugins/models/vae.py

@@ -88,6 +88,6 @@ def vae_loss(x, x_decoded_mean):
 
 
 class ModelClass(DeepProfilerModel):
-    def __init__(self, config, dset, generator, val_generator):
-        super(ModelClass, self).__init__(config, dset, generator, val_generator)
+    def __init__(self, config, dset, gpu, generator, val_generator):
+        super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator)
         self.feature_model, self.encoder, self.generator, self.optimizer, self.loss = define_model(config, dset)

tests/deepprofiler/learning/test_model.py

@@ -111,19 +111,22 @@ def val_crop_generator(config):
     generator = module.SingleImageGeneratorClass
     return generator
 
+@pytest.fixture(scope="function")
+def gpu():
+    return '0'
 
 @pytest.fixture(scope="function")
-def model(config, dataset, crop_generator, val_crop_generator):
+def model(config, dataset, gpu, crop_generator, val_crop_generator):
     def create():
         module = importlib.import_module("plugins.models.{}".format(config["train"]["model"]["name"]))
         importlib.invalidate_caches()
-        dpmodel = module.ModelClass(config, dataset, crop_generator, val_crop_generator)
+        dpmodel = module.ModelClass(config, dataset, gpu, crop_generator, val_crop_generator)
         return dpmodel
     return create
 
 
-def test_init(config, dataset, crop_generator, val_crop_generator):
-    dpmodel = DeepProfilerModel(config, dataset, crop_generator, val_crop_generator)
+def test_init(config, dataset, gpu, crop_generator, val_crop_generator):
+    dpmodel = DeepProfilerModel(config, dataset, gpu, crop_generator, val_crop_generator)
     assert dpmodel.feature_model is None
     assert dpmodel.config == config
     assert dpmodel.dset == dataset

tests/deepprofiler/learning/test_profiling.py

@@ -98,6 +98,12 @@ def locations(out_dir, metadata, config, make_struct):
         })
         locs.to_csv(path, index=False)
 
+
+@pytest.fixture(scope="function")
+def gpu():
+    return '0'
+
+
 @pytest.fixture(scope="function")
 def checkpoint(config, dataset):
     crop_generator = importlib.import_module(
@@ -107,20 +113,20 @@ def checkpoint(config, dataset):
         "plugins.crop_generators.{}".format(config["train"]["model"]["crop_generator"])) \
         .SingleImageGeneratorClass
     dpmodel = importlib.import_module("plugins.models.{}".format(config["train"]["model"]["name"])) \
-        .ModelClass(config, dataset, crop_generator, profile_crop_generator)
+        .ModelClass(config, dataset, gpu, crop_generator, profile_crop_generator)
     dpmodel.feature_model.compile(dpmodel.optimizer, dpmodel.loss)
     filename = os.path.join(config["paths"]["checkpoints"], config["profile"]["checkpoint"])
     dpmodel.feature_model.save_weights(filename)
     return filename
 
 
 @pytest.fixture(scope="function")
-def profile(config, dataset):
-    return deepprofiler.learning.profiling.Profile(config, dataset)
+def profile(config, dataset, gpu):
+    return deepprofiler.learning.profiling.Profile(config, dataset, gpu)
 
 
-def test_init(config, dataset):
-    prof = deepprofiler.learning.profiling.Profile(config, dataset)
+def test_init(config, dataset, gpu):
+    prof = deepprofiler.learning.profiling.Profile(config, dataset, gpu)
     test_num_channels = len(config["dataset"]["images"]["channels"])
     assert prof.config == config
     assert prof.dset == dataset
@@ -152,8 +158,8 @@ def test_extract_features(profile, metadata, locations, checkpoint):
     assert os.path.isfile(output_file)
 
 
-def test_profile(config, dataset, data, locations, checkpoint):
-    deepprofiler.learning.profiling.profile(config, dataset)
+def test_profile(config, dataset, data, locations, checkpoint, gpu):
+    deepprofiler.learning.profiling.profile(config, dataset, gpu)
     for index, row in dataset.meta.data.iterrows():
         output_file = config["paths"]["features"] + "/{}_{}_{}.npz" \
             .format(row["Metadata_Plate"], row["Metadata_Well"], row["Metadata_Site"])

tests/deepprofiler/learning/test_training.py

@@ -98,13 +98,14 @@ def locations(out_dir, metadata, config, make_struct):
 def test_learn_model(config, dataset, data, locations, out_dir, make_struct):
     epoch = 1
     verbose = 1
-    deepprofiler.learning.training.learn_model(config, dataset, epoch, verbose=verbose)
+    gpu = '0'
+    deepprofiler.learning.training.learn_model(config, dataset, gpu, epoch, verbose=verbose)
     assert os.path.exists(os.path.join(config["paths"]["checkpoints"], "checkpoint_0001.hdf5"))
     assert os.path.exists(os.path.join(config["paths"]["checkpoints"], "checkpoint_0002.hdf5"))
     assert os.path.exists(os.path.join(config["paths"]["logs"], "log.csv"))
     epoch = 3
     config["train"]["model"]["epochs"] = 4
-    deepprofiler.learning.training.learn_model(config, dataset, epoch, verbose=verbose)
+    deepprofiler.learning.training.learn_model(config, dataset, gpu, epoch, verbose=verbose)
     assert os.path.exists(os.path.join(config["paths"]["checkpoints"], "checkpoint_0003.hdf5"))
     assert os.path.exists(os.path.join(config["paths"]["checkpoints"], "checkpoint_0004.hdf5"))
     assert os.path.exists(os.path.join(config["paths"]["logs"], "log.csv"))

tests/plugins/models/test_autoencoder.py

@@ -82,6 +82,10 @@ def val_generator():
     return deepprofiler.imaging.cropping.SingleImageCropGenerator
 
 
+@pytest.fixture(scope="function")
+def gpu():
+    return '0'
+
 def test_define_model(config, dataset):
     autoencoder, encoder, decoder, optimizer, loss = plugins.models.autoencoder.define_model(config, dataset)
     assert isinstance(autoencoder, keras.Model)
@@ -91,8 +95,8 @@ def test_define_model(config, dataset):
     assert isinstance(loss, str) or callable(loss)
 
 
-def test_init(config, dataset, generator, val_generator):
-    dpmodel = plugins.models.autoencoder.ModelClass(config, dataset, generator, val_generator)
+def test_init(config, dataset, gpu, generator, val_generator):
+    dpmodel = plugins.models.autoencoder.ModelClass(config, dataset, gpu, generator, val_generator)
     autoencoder, encoder, decoder, optimizer, loss = plugins.models.autoencoder.define_model(config, dataset)
     assert dpmodel.feature_model.__eq__(autoencoder)
     assert dpmodel.encoder.__eq__(encoder)