12
12
13
13
try :
14
14
import psutil
15
- import memory_profiler
16
15
run_profiler = True
17
16
skip_profile_msg = 'Run profiler tests'
18
17
except ImportError as exc :
@@ -28,9 +27,9 @@ class UseResourcesInputSpec(CommandLineInputSpec):
28
27
29
28
# Init attributes
30
29
num_gb = traits .Float (desc = 'Number of GB of RAM to use' ,
31
- argstr = " -g %f" )
30
+ argstr = ' -g %f' )
32
31
num_procs = traits .Int (desc = 'Number of processors to use' ,
33
- argstr = " -p %d" )
32
+ argstr = ' -p %d' )
34
33
35
34
36
35
# UseResources interface
@@ -78,30 +77,21 @@ def _use_gb_ram(num_gb):
78
77
del gb_str
79
78
80
79
# Import packages
81
- import logging
82
- from multiprocessing import Process
83
-
84
80
from threading import Thread
85
81
86
82
# Init variables
87
83
num_gb = float (num_gb )
88
- # Init variables
89
- #num_threads = proc.num_threads()
90
- from CPAC .utils .utils import setup_logger
84
+
91
85
# Build proc list
92
86
proc_list = []
93
87
for idx in range (num_procs ):
94
- #proc = Thread(target=_use_gb_ram, args=(num_gb/num_procs,), name=str(idx))
95
- proc = Process (target = _use_gb_ram , args = (num_gb / num_procs ,), name = str (idx ))
88
+ proc = Thread (target = _use_gb_ram , args = (num_gb / num_procs ,), name = str (idx ))
96
89
proc_list .append (proc )
97
90
98
- logger = setup_logger ('memory_profiler' , '/home/dclark/memory_profiler.log' ,
99
- logging .DEBUG , to_screen = False )
100
91
# Run multi-threaded
101
- print 'Using %.3f GB of memory over %d processors ...' % (num_gb , num_procs )
92
+ print 'Using %.3f GB of memory over %d sub-threads ...' % (num_gb , num_procs )
102
93
for idx , proc in enumerate (proc_list ):
103
94
proc .start ()
104
- #logger.debug('Starting PID: %d' % proc.pid)
105
95
106
96
for proc in proc_list :
107
97
proc .join ()
@@ -137,9 +127,9 @@ def setUp(self):
137
127
138
128
# Init parameters
139
129
# Input RAM GB to occupy
140
- self .num_gb = 4
141
- # Input number of processors
142
- self .num_procs = 1
130
+ self .num_gb = 6
131
+ # Input number of sub-threads (not including parent threads)
132
+ self .num_threads = 7
143
133
# Acceptable percent error for memory profiled against input
144
134
self .mem_err_percent = 5
145
135
@@ -367,30 +357,33 @@ def test_cmdline_profiling(self):
367
357
368
358
# Init variables
369
359
num_gb = self .num_gb
370
- num_procs = self .num_procs
360
+ num_threads = self .num_threads
371
361
372
362
# Run workflow and get stats
373
- finish_str = self ._run_cmdline_workflow (num_gb , num_procs )
363
+ finish_str = self ._run_cmdline_workflow (num_gb , num_threads )
374
364
# Get runtime stats as dictionary
375
365
node_stats = json .loads (finish_str )
376
366
377
367
# Read out runtime stats
378
368
runtime_gb = float (node_stats ['runtime_memory_gb' ])
379
- runtime_procs = int (node_stats ['runtime_threads' ])
369
+ runtime_threads = int (node_stats ['runtime_threads' ])
380
370
381
371
# Get margin of error for RAM GB
382
372
allowed_gb_err = (self .mem_err_percent / 100.0 )* num_gb
383
373
runtime_gb_err = np .abs (runtime_gb - num_gb )
374
+ # Runtime threads should reflect shell-cmd thread, Python parent thread
375
+ # and Python sub-threads = 1 + 1 + num_threads
376
+ expected_runtime_threads = 1 + 1 + num_threads
384
377
385
378
# Error message formatting
386
379
mem_err = 'Input memory: %f is not within %.1f%% of runtime ' \
387
380
'memory: %f' % (num_gb , self .mem_err_percent , runtime_gb )
388
- procs_err = 'Input procs : %d is not equal to runtime procs : %d' \
389
- % (num_procs , runtime_procs )
381
+ procs_err = 'Input threads : %d is not equal to runtime threads : %d' \
382
+ % (expected_runtime_threads , runtime_threads )
390
383
391
384
# Assert runtime stats are what was input
392
385
self .assertLessEqual (runtime_gb_err , allowed_gb_err , msg = mem_err )
393
- self .assertEqual (num_procs , runtime_procs , msg = procs_err )
386
+ self .assertEqual (expected_runtime_threads , runtime_threads , msg = procs_err )
394
387
395
388
# Test resources were used as expected
396
389
@unittest .skipIf (run_profiler == False , skip_profile_msg )
@@ -406,30 +399,33 @@ def test_function_profiling(self):
406
399
407
400
# Init variables
408
401
num_gb = self .num_gb
409
- num_procs = self .num_procs
402
+ num_threads = self .num_threads
410
403
411
404
# Run workflow and get stats
412
- finish_str = self ._run_function_workflow (num_gb , num_procs )
405
+ finish_str = self ._run_function_workflow (num_gb , num_threads )
413
406
# Get runtime stats as dictionary
414
407
node_stats = json .loads (finish_str )
415
408
416
409
# Read out runtime stats
417
410
runtime_gb = float (node_stats ['runtime_memory_gb' ])
418
- runtime_procs = int (node_stats ['runtime_threads' ])
411
+ runtime_threads = int (node_stats ['runtime_threads' ])
419
412
420
413
# Get margin of error for RAM GB
421
414
allowed_gb_err = (self .mem_err_percent / 100.0 )* num_gb
422
415
runtime_gb_err = np .abs (runtime_gb - num_gb )
416
+ # Runtime threads should reflect Python parent thread
417
+ # and Python sub-threads = 1 + num_threads
418
+ expected_runtime_threads = 1 + num_threads
423
419
424
420
# Error message formatting
425
421
mem_err = 'Input memory: %f is not within %.1f%% of runtime ' \
426
422
'memory: %f' % (num_gb , self .mem_err_percent , runtime_gb )
427
423
procs_err = 'Input procs: %d is not equal to runtime procs: %d' \
428
- % (num_procs , runtime_procs )
424
+ % (expected_runtime_threads , runtime_threads )
429
425
430
426
# Assert runtime stats are what was input
431
427
self .assertLessEqual (runtime_gb_err , allowed_gb_err , msg = mem_err )
432
- self .assertEqual (num_procs , runtime_procs , msg = procs_err )
428
+ self .assertEqual (expected_runtime_threads , runtime_threads , msg = procs_err )
433
429
434
430
435
431
# Command-line run-able unittest module
0 commit comments