|
| 1 | +# Copyright (C) 2024 |
| 2 | +# Associated Universities, Inc. Washington DC, USA. |
| 3 | +# |
| 4 | +# This library is free software; you can redistribute it and/or modify it |
| 5 | +# under the terms of the GNU Library General Public License as published by |
| 6 | +# the Free Software Foundation; either version 2 of the License, or (at your |
| 7 | +# option) any later version. |
| 8 | +# |
| 9 | +# This library is distributed in the hope that it will be useful, but WITHOUT |
| 10 | +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public |
| 12 | +# License for more details.is |
| 13 | +# |
| 14 | +# You should have received a copy of the GNU Library General Public License |
| 15 | +# along with this library; if not, write to the Free Software Foundation, |
| 16 | +# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA. |
| 17 | +# |
| 18 | +# Correspondence concerning this should be addressed as follows: |
| 19 | +# Postal address: National Radio Astronomy Observatory |
| 20 | +# 1003 Lopezville Road, |
| 21 | +# Socorro, NM - 87801, USA |
| 22 | +# |
| 23 | +# $Id$ |
| 24 | + |
| 25 | +import os |
| 26 | +from subprocess import Popen,PIPE |
| 27 | + |
| 28 | +class GPUMonitor(object): |
| 29 | + |
| 30 | + def __init__(self): |
| 31 | + self.device = self.getDevice() |
| 32 | + self.GPUName = self.getDeviceProperty('name') |
| 33 | + self.GPUCapability = self.getDeviceProperty('compute_cap') |
| 34 | + self.listSystemGPUs() |
| 35 | + |
| 36 | + def listSystemGPUs(self): |
| 37 | + with Popen(['nvidia-smi'], stdout = PIPE) as nvidia_smi: |
| 38 | + print(nvidia_smi.stdout.read().decode('ascii')) |
| 39 | + |
| 40 | + def getDevice(self): |
| 41 | + try: |
| 42 | + device = os.environ['NVIDIA_VISIBLE_DEVICES'] |
| 43 | + except KeyError: |
| 44 | + print('Variable ${NVIDIA_VISIBLE_DEVICES} not set, obtaining device id with uuid. This may be a problem in systems with multiple GPUs.') |
| 45 | + cmd = ['nvidia-smi','--query-gpu=uuid','--format=csv,noheader'] |
| 46 | + with Popen(cmd, stdout = PIPE) as nvidia_smi: |
| 47 | + device = nvidia_smi.stdout.read().decode('ascii').strip() |
| 48 | + except Exception as e: |
| 49 | + raise e |
| 50 | + return device |
| 51 | + |
| 52 | + def getDeviceProperty(self, property): |
| 53 | + cmd = ['nvidia-smi','--query-gpu=' + property,'--id=' + self.device,'--format=csv,noheader'] |
| 54 | + with Popen(cmd, stdout = PIPE) as nvidia_smi: |
| 55 | + devprop = nvidia_smi.stdout.read().decode('ascii').strip() |
| 56 | + return devprop |
| 57 | + |
| 58 | + def startMonitor(self, interval = 1, outputfile = 'GPUMonitor.out'): |
| 59 | + cmd = ['nvidia-smi','--query-gpu=timestamp,name,utilization.gpu,utilization.memory,memory.used', |
| 60 | + '--format=csv','-l',str(interval),'--id=' + self.device,'--filename=' + outputfile] |
| 61 | + self.monitor = Popen(cmd) |
| 62 | + |
| 63 | + def stopMonitor(self): |
| 64 | + self.monitor.send_signal(2) |
0 commit comments