Skip to content

Commit f76043c

Browse files
friskluftfriskluft
authored
POL5598 SLURM-friendliness for CLI (PolusAI#248)
1 parent e0366b4 commit f76043c

File tree

10 files changed

+299
-93
lines changed

10 files changed

+299
-93
lines changed

‎CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,7 @@ set(SOURCE
192192
src/nyx/cli_fpimage_options.cpp
193193
src/nyx/cli_gabor_options.cpp
194194
src/nyx/cli_glcm_options.cpp
195+
src/nyx/cli_gpu_options.cpp
195196
src/nyx/cli_nested_roi_options.cpp
196197
src/nyx/common_stats.cpp
197198
src/nyx/dirs_and_files.cpp

‎README.md‎

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -358,23 +358,25 @@ Assuming you [built the Nyxus binary](#building-from-source) as outlined below,
358358
--intDir | Directory of intensity image collection | path
359359
--outDir | Output directory | path
360360
--segDir | Directory of labeled image collection | path
361-
--coarseGrayDepth | (optional) Custom number of greyscale level bins used in texture features. Default: '--coarseGrayDepth=256' | integer
362-
--glcmAngles | (optional) Enabled direction angles of the GLCM feature. Superset of values: 0, 45, 90, and 135. Default: '--glcmAngles=0,45,90,135' | list of integer constants
363-
--intSegMapDir | (optional) Data collection of the ad-hoc intensity-to-mask file mapping. Must be used in combination with parameter '--intSegMapFile' | path
364-
--intSegMapFile | (optional) Name of the text file containing an ad-hoc intensity-to-mask file mapping. The files are assumed to reside in corresponding intensity and label collections. Must be used in combination with parameter '--intSegMapDir' | string
365-
--pixelDistance | (optional) Number of pixels to treat ROIs within specified distance as neighbors. Default value: '--pixelDistance=5' | integer
366-
--pixelsPerCentimeter | (optional) Number of pixels in centimeter used by unit length-related features. Default value: 0 | real
367-
--ramLimit | (optional) Amount of memory not to exceed by Nyxus, in megabytes. Default value: 50\% of available memory. Example: '--ramLimit=2000' to use 2,000 megabytes | integer
368-
--reduceThreads | (optional) Number of CPU threads used on the feature calculation step. Default: '--reduceThreads=1' | integer
369-
--skiproi | (optional) Skip ROIs having specified labels. Example: '--skiproi=image1.tif:2,3,4;image2.tif:45,56' | string
370-
--tempDir | (optional) Directory used by temporary out-of-RAM objects. Default value: system temporary directory | path
371-
--hsig | (optional) Channel signature Example: "--hsig=_c" to match images whose file names have channel info starting substring '_c' like in 'p0_y1_r1_c1.ome.tiff' | string
372-
--hpar | (optional) Channel number that should be used as a provider of parent segments. Example: '--hpar=1' | integer
373-
--hchi | (optional) Channel number that should be used as a provider of child segments. Example: '--hchi=0' | integer
374-
--hag | (optional) Name of a method how to aggregate features of segments recognized as children of same parent segment. Valid options are 'SUM', 'MEAN', 'MIN', 'MAX', 'WMA' (weighted mean average), and 'NONE' (no aggregation, instead, same parent child segments will be laid out horizontally) | string
375-
--fpimgdr | (optional) Desired dynamic range of voxels of a floating point TIFF image. Example: --fpimgdr=240 makes intensities be read in range 0-240. Default value: 10e4 | unsigned integer
376-
--fpimgmin | (optional) Minimum intensity of voxels of a floating point TIFF image. Default value: 0.0 | real
377-
--fpimgdr | (optional) Maximum intensity of voxels of a floating point TIFF image. Default value: 1.0 | real
361+
--useGpu | ${\color{red}\textsf{(optional)}}$ Calculate compute-expensive features on an NVIDIA GPU device specified by parameter --gpuDeviceID. Default: '--useGpu=false'. Example: --useGpu=true | boolean
362+
--gpuDeviceID | ${\color{red}\textsf{(optional)}}$ ID of a GPU device to be used when '--useGpu=true'. Default: '--gpuDeviceID=0'. Example 1 (single GPU device): '--useGpu=true --gpuDeviceID=2' to strictly use device 2. Example 2 (multiple GPU devices, usually in SLURM scenarios): '--useGpu=true --gpuDeviceID=0,1,3' to use the GPU device having maximum free RAM of devices 0, 1, and 3. | integer or list of integers
363+
--coarseGrayDepth | ${\color{red}\textsf{(optional)}}$ Custom number of greyscale level bins used in texture features. Default: '--coarseGrayDepth=256' | integer
364+
--glcmAngles | ${\color{red}\textsf{(optional)}}$ Enabled direction angles of the GLCM feature. Superset of values: 0, 45, 90, and 135. Default: '--glcmAngles=0,45,90,135' | list of integers
365+
--intSegMapDir | ${\color{red}\textsf{(optional)}}$ Data collection of the ad-hoc intensity-to-mask file mapping. Must be used in combination with parameter '--intSegMapFile' | path
366+
--intSegMapFile | ${\color{red}\textsf{(optional)}}$ Name of the text file containing an ad-hoc intensity-to-mask file mapping. The files are assumed to reside in corresponding intensity and label collections. Must be used in combination with parameter '--intSegMapDir' | string
367+
--pixelDistance | ${\color{red}\textsf{(optional)}}$ Number of pixels to treat ROIs within specified distance as neighbors. Default value: '--pixelDistance=5' | integer
368+
--pixelsPerCentimeter | ${\color{red}\textsf{(optional)}}$ Number of pixels in centimeter used by unit length-related features. Default value: 0 | real
369+
--ramLimit | ${\color{red}\textsf{(optional)}}$ Amount of memory not to exceed by Nyxus, in megabytes. Default value: 50\% of available memory. Example: '--ramLimit=2000' to use 2,000 megabytes | integer
370+
--reduceThreads | ${\color{red}\textsf{(optional)}}$ Number of CPU threads used on the feature calculation step. Default: '--reduceThreads=1' | integer
371+
--skiproi | ${\color{red}\textsf{(optional)}}$ Skip ROIs having specified labels. Example: '--skiproi=image1.tif:2,3,4;image2.tif:45,56' | string
372+
--tempDir | ${\color{red}\textsf{(optional)}}$ Directory used by temporary out-of-RAM objects. Default value: system temporary directory | path
373+
--hsig | ${\color{red}\textsf{(optional)}}$ Channel signature Example: "--hsig=_c" to match images whose file names have channel info starting substring '_c' like in 'p0_y1_r1_c1.ome.tiff' | string
374+
--hpar | ${\color{red}\textsf{(optional)}}$ Channel number that should be used as a provider of parent segments. Example: '--hpar=1' | integer
375+
--hchi | ${\color{red}\textsf{(optional)}}$ Channel number that should be used as a provider of child segments. Example: '--hchi=0' | integer
376+
--hag | ${\color{red}\textsf{(optional)}}$ Name of a method how to aggregate features of segments recognized as children of same parent segment. Valid options are 'SUM', 'MEAN', 'MIN', 'MAX', 'WMA' (weighted mean average), and 'NONE' (no aggregation, instead, same parent child segments will be laid out horizontally) | string
377+
--fpimgdr | ${\color{red}\textsf{(optional)}}$ Desired dynamic range of voxels of a floating point TIFF image. Example: --fpimgdr=240 makes intensities be read in range 0-240. Default value: 10e4 | unsigned integer
378+
--fpimgmin | ${\color{red}\textsf{(optional)}}$ Minimum intensity of voxels of a floating point TIFF image. Default value: 0.0 | real
379+
--fpimgdr | ${\color{red}\textsf{(optional)}}$ Maximum intensity of voxels of a floating point TIFF image. Default value: 1.0 | real
378380

379381
---
380382

@@ -595,7 +597,7 @@ These packages also have underlying dependencies and at times, these dependency
595597

596598
By default, Nyxus can be built with a minimal set of dependecies (Tiff support and Python interface). To build Nyxus with all the supported IO options mentioned above, pass `-DALLEXTRAS=ON` in the `cmake` command.
597599

598-
### __GPU Support__
600+
### __Adding GPU Support__
599601
Nyxus also can be build with NVIDIA GPU support. To do so, a `CUDA` Development toolkit compatible with the host `C++` compiler need to be present in the system. For building with GPU support, pass `-DUSEGPU=ON` flag in the `cmake` command.
600602

601603
### __Inside Conda__

‎src/nyx/cli_gpu_options.cpp‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
#include "cli_gpu_options.h"
2+
#include "helpers/helpers.h"
3+
4+
#ifdef USE_GPU
5+
namespace NyxusGpu
6+
{
7+
bool get_best_device(
8+
// in
9+
const std::vector<int>& devIds,
10+
// out
11+
int& best_id,
12+
std::string& lastCuErmsg);
13+
}
14+
#endif
15+
16+
bool GpuOptions::empty()
17+
{
18+
return raw_use_gpu.empty();
19+
}
20+
21+
void GpuOptions::set_using_gpu(bool use)
22+
{
23+
using_gpu_ = use;
24+
}
25+
26+
bool GpuOptions::get_using_gpu()
27+
{
28+
return using_gpu_;
29+
}
30+
31+
bool GpuOptions::set_single_device_id(int id)
32+
{
33+
if (get_using_gpu())
34+
{
35+
this->best_device_id_ = id;
36+
return true;
37+
}
38+
else
39+
return false;
40+
}
41+
42+
int GpuOptions::get_single_device_id()
43+
{
44+
return best_device_id_;
45+
}
46+
47+
bool GpuOptions::parse_input (std::string & ermsg)
48+
{
49+
#ifdef USE_GPU
50+
51+
auto u = Nyxus::toupper (this->raw_use_gpu);
52+
if (u.length() == 0)
53+
{
54+
set_using_gpu (false);
55+
}
56+
else
57+
{
58+
auto t = Nyxus::toupper("true"),
59+
f = Nyxus::toupper("false");
60+
if (u != t && u != f)
61+
{
62+
ermsg = "valid values are " + t + " or " + f;
63+
return false;
64+
}
65+
set_using_gpu (u == t);
66+
67+
// process user's GPU device choice
68+
if (get_using_gpu())
69+
{
70+
std::vector<int> devIds;
71+
72+
if (! this->raw_requested_device_ids.empty())
73+
{
74+
// user input -> vector of IDs
75+
std::vector<std::string> S;
76+
Nyxus::parse_delimited_string (this->raw_requested_device_ids, ",", S);
77+
// examine those IDs
78+
for (const auto& s : S)
79+
{
80+
if (!s.empty())
81+
{
82+
// string -> int
83+
int id;
84+
if (sscanf (s.c_str(), "%d", &id) != 1 || id < 0)
85+
{
86+
ermsg = s + ": expecting a non-negative integer";
87+
return false;
88+
}
89+
devIds.push_back (id);
90+
}
91+
}
92+
}
93+
else
94+
devIds.push_back (0); // user did not requested a specific ID, so default it to 0
95+
96+
// given a set of suggested devices, choose the least memory-busy one
97+
int best_id = -1;
98+
std::string lastCuErmsg;
99+
if (! NyxusGpu::get_best_device(devIds, best_id, lastCuErmsg))
100+
{
101+
ermsg = "cannot use any of GPU devices in " + Nyxus::virguler<int> (devIds) + " due to " + lastCuErmsg;
102+
return false;
103+
}
104+
105+
// we found at least one workable
106+
this->best_device_id_ = best_id;
107+
}
108+
}
109+
110+
return true;
111+
112+
#else
113+
114+
ermsg = "To have GPU options available, use a Nyxus version with GPU support enabled";
115+
set_using_gpu (false);
116+
return false;
117+
118+
#endif
119+
}

‎src/nyx/cli_gpu_options.h‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#pragma once
2+
3+
#include <string>
4+
#include <vector>
5+
6+
class GpuOptions
7+
{
8+
public:
9+
// parses "raw_use_gpu" and "raw_requested_device_ids"
10+
bool parse_input (std::string & ermsg);
11+
12+
// true if the parameters have never been specified via "raw_*"
13+
bool empty();
14+
15+
// accessor of the "using" status
16+
void set_using_gpu(bool use);
17+
bool get_using_gpu();
18+
19+
// accessor of active device ID
20+
bool set_single_device_id(int id);
21+
int get_single_device_id();
22+
23+
// exposed to command line processor
24+
std::string raw_use_gpu;
25+
std::string raw_requested_device_ids;
26+
27+
private:
28+
bool using_gpu_ = false;
29+
int best_device_id_ = -1;
30+
};

‎src/nyx/environment.cpp‎

Lines changed: 34 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@
1414
#include "helpers/timing.h"
1515
#include "version.h"
1616

17+
#ifdef USE_GPU
18+
std::vector<std::map<std::string, std::string>> get_gpu_properties();
19+
#endif
20+
1721
namespace Nyxus
1822
{
1923
bool existsOnFilesystem(const std::string&);
@@ -172,7 +176,7 @@ void Environment::show_cmdline_help()
172176
<< "\tnyxus --help\tDisplay help info\n";
173177

174178
#ifdef USE_GPU
175-
std::cout << " [" << USEGPU << "=<true or false>" << " [" << GPUDEVICEID << "=<valid GPU device ID>] ]\n";
179+
std::cout << " [" << USEGPU << "=<true or false>" << " [" << GPUDEVICEID << "=<comma separated GPU device ID>] ]\n";
176180
#endif
177181
}
178182

@@ -197,9 +201,10 @@ void Environment::show_summary(const std::string& head, const std::string& tail)
197201
<< "\tverbosity level\t" << verbosity_level << "\n";
198202

199203
#ifdef USE_GPU
200-
std::cout << "\tusing GPU\t" << (using_gpu() ? "yes" : "no") << "\n";
201-
if (using_gpu())
202-
std::cout << "\tGPU device ID \t" << get_gpu_device_choice() << "\n";
204+
std::cout << "\tusing GPU\t" << (gpuOptions.get_using_gpu() ? "yes" : "no") << "\n";
205+
if (gpuOptions.get_using_gpu())
206+
std::cout << "\trequested GPU device IDs: " << (gpuOptions.raw_requested_device_ids.empty() ? "(blank)" : gpuOptions.raw_requested_device_ids) << "\n"
207+
"\tbest GPU device ID: " << gpuOptions.get_single_device_id() << "\n";
203208
#endif
204209

205210
// Features
@@ -388,8 +393,8 @@ bool Environment::parse_cmdline(int argc, char** argv)
388393
#endif
389394

390395
#ifdef USE_GPU
391-
|| find_string_argument(i, USEGPU, rawUseGpu)
392-
|| find_string_argument(i, GPUDEVICEID, rawGpuDeviceID)
396+
|| find_string_argument(i, USEGPU, gpuOptions.raw_use_gpu)
397+
|| find_string_argument(i, GPUDEVICEID, gpuOptions.raw_requested_device_ids)
393398
#endif
394399
))
395400
unrecognizedArgs.push_back(*i);
@@ -700,7 +705,7 @@ bool Environment::parse_cmdline(int argc, char** argv)
700705
}
701706

702707
//==== Parse exclusive-inclusive timing
703-
#ifdef CHECKTIMING
708+
#ifdef CHECKTIMING
704709
if (!rawExclusiveTiming.empty())
705710
{
706711
std::transform(rawExclusiveTiming.begin(), rawExclusiveTiming.end(), rawExclusiveTiming.begin(), ::tolower);
@@ -709,44 +714,20 @@ bool Environment::parse_cmdline(int argc, char** argv)
709714
else
710715
Stopwatch::set_inclusive(true);
711716
}
712-
#endif
717+
#endif
713718

714719
//==== Using GPU
715-
#ifdef USE_GPU
716-
auto rawUseGpuUC = Nyxus::toupper(rawUseGpu);
717-
if (rawUseGpuUC.length() == 0)
718-
{
719-
set_use_gpu(false);
720-
std::cout << "\n!\n! Not using GPU. To involve GPU, use command line option " << USEGPU << "=true\n!\n\n";
721-
}
722-
else
720+
#ifdef USE_GPU
721+
if (!gpuOptions.empty())
723722
{
724-
auto validUsegpu1 = Nyxus::toupper("true"),
725-
validUsegpu2 = Nyxus::toupper("false");
726-
if (rawUseGpuUC != validUsegpu1 && rawUseGpuUC != validUsegpu2)
723+
std::string ermsg;
724+
if (!gpuOptions.parse_input(ermsg))
727725
{
728-
std::cerr << "Error: valid values of " << USEGPU << " are " << validUsegpu1 << " or " << validUsegpu2 << "\n";
726+
std::cerr << ermsg << "\n";
729727
return false;
730728
}
731-
use_gpu_ = rawUseGpuUC == validUsegpu1;
732-
733-
// Process user's GPU device choice
734-
if (use_gpu_)
735-
{
736-
if (!rawGpuDeviceID.empty())
737-
{
738-
// string -> integer
739-
if (sscanf(rawGpuDeviceID.c_str(), "%d", &gpu_device_id_) != 1 || gpu_device_id_ < 0)
740-
{
741-
std::cerr << "Error: " << GPUDEVICEID << "=" << gpu_device_id_ << ": expecting 0 or positive integer constant\n";
742-
return false;
743-
}
744-
}
745-
else
746-
gpu_device_id_ = 0; // Specific GPU device ID was not requested, defaulting to 0
747-
}
748729
}
749-
#endif
730+
#endif
750731

751732
//==== Parse desired features
752733

@@ -970,48 +951,50 @@ bool Environment::arrow_is_enabled()
970951

971952
void Environment::set_gpu_device_id (int choice)
972953
{
973-
auto n_gpus = get_gpu_properties().size();
974-
if (n_gpus == 0)
954+
auto prp = get_gpu_properties();
955+
auto n= prp.size();
956+
if (n == 0)
975957
{
976958
std::cerr << "Error: no GPU devices available \n";
977959
return;
978960
}
979961

980-
if (choice > get_gpu_properties().size() - 1)
962+
if (choice >= n)
981963
{
982964
std::cerr << "Warning: GPU choice (" << choice << ") is out of range. Defaulting to device 0 \n";
983-
gpu_device_id_ = 0;
984-
return;
965+
gpuOptions.set_single_device_id (0);
985966
}
986-
987-
gpu_device_id_ = choice;
967+
else
968+
gpuOptions.set_single_device_id (choice);
988969
}
989970

990971
int Environment::get_gpu_device_choice()
991972
{
992973
if (using_gpu())
993-
return gpu_device_id_;
974+
return gpuOptions.get_single_device_id();
994975
else
995976
return -1; // GPU was not requested so return an invalid device ID -1
996977
}
997978

998-
void Environment::set_use_gpu(bool yes)
979+
void Environment::set_using_gpu (bool yes)
999980
{
1000-
use_gpu_ = yes;
981+
gpuOptions.set_using_gpu (yes);
1001982
}
1002983

1003984
bool Environment::using_gpu()
1004985
{
1005-
return use_gpu_;
986+
return gpuOptions.get_using_gpu();
1006987
}
1007988

1008-
std::vector<std::map<std::string, std::string>> Environment::get_gpu_properties() {
989+
std::vector<std::map<std::string, std::string>> Environment::get_gpu_properties()
990+
{
1009991
int n_devices;
1010992
std::vector<std::map<std::string, std::string>> props;
1011993

1012994
cudaGetDeviceCount(&n_devices);
1013995

1014-
for (int i = 0; i < n_devices; ++i) {
996+
for (int i = 0; i < n_devices; ++i)
997+
{
1015998
cudaDeviceProp prop;
1016999
cudaGetDeviceProperties(&prop, i);
10171000

0 commit comments

Comments
 (0)