From 77a8e8bb9ce869782e2f7019bcc50259d2a60e7d Mon Sep 17 00:00:00 2001 From: Maxwell Alexander Xu Date: Thu, 5 Oct 2023 17:04:38 -0400 Subject: [PATCH 1/2] tweaked the gpu_users alias to allow for an input of a specific lab, so the output table would be filtered to only have that lab --- README.md | 8 ++++++++ gpu_types_usage.awk | 2 +- gpu_types_usage_verbose.awk | 2 +- gpus_users.bashrc | 15 +++++++-------- lab_usage.awk | 7 ++++++- lab_usage_qos.awk | 7 ++++++- lab_usage_verbose.awk | 7 ++++++- node_usage.awk | 2 +- 8 files changed, 36 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index fb40149..eb91222 100644 --- a/README.md +++ b/README.md @@ -37,3 +37,11 @@ gpus_users -v # Summary of node usage node_usage ``` + +Any of the gpu_users command can be followed up with a lab (e.g. cvmlp, rehg, hoffman, etc.) to filter for the usage only in that lab, i.e. + +``` +gpus_users --cvmlp +gpus_users -q --cvmlp +gpus_users -v --cvmlp +``` diff --git a/gpu_types_usage.awk b/gpu_types_usage.awk index c0a7871..549c17a 100644 --- a/gpu_types_usage.awk +++ b/gpu_types_usage.awk @@ -1,5 +1,5 @@ #!/bin/awk -f -@include "mycolors.awk" +@include "/nethome/mxu87/slurm_usage_utils/mycolors.awk" BEGIN { FS="[ :|=,]"; diff --git a/gpu_types_usage_verbose.awk b/gpu_types_usage_verbose.awk index 19fe122..f779877 100644 --- a/gpu_types_usage_verbose.awk +++ b/gpu_types_usage_verbose.awk @@ -1,5 +1,5 @@ #!/bin/awk -f -@include "mycolors.awk" +@include "/nethome/mxu87/slurm_usage_utils/mycolors.awk" BEGIN { FS="[ :|=,]"; diff --git a/gpus_users.bashrc b/gpus_users.bashrc index 17c4905..8a2138a 100644 --- a/gpus_users.bashrc +++ b/gpus_users.bashrc @@ -3,7 +3,7 @@ usage_by_lab() { { sacctmgr -nop show assoc format=account,user,grptres | grep -v 'root' | grep -v 'test-lab'; squeue -O "UserName,StateCompact,QOS,tres-alloc:1000,Account,Partition" -h | tr -s " " | awk '$0="G> "$0' | grep gpu | sort; - } | awk -f $1 - + } | awk -v labfilter="$2" -f $1 - } usage_by_node() { @@ -16,15 +16,14 @@ usage_by_node_type() { } | awk -f $1 - } + gpus_users() { - if [ $# -eq 1 ]; then - if [[ $1 == "-q" ]]; then - usage_by_lab ~/slurm_usage_utils/lab_usage_qos.awk - elif [[ $1 == "-v" ]]; then - usage_by_lab ~/slurm_usage_utils/lab_usage_verbose.awk - fi + if [[ $1 == "-q" ]]; then + usage_by_lab ~/slurm_usage_utils/lab_usage_qos.awk $2 + elif [[ $1 == "-v" ]]; then + usage_by_lab ~/slurm_usage_utils/lab_usage_verbose.awk $2 else - usage_by_lab ~/slurm_usage_utils/lab_usage.awk + usage_by_lab ~/slurm_usage_utils/lab_usage.awk $1 fi } diff --git a/lab_usage.awk b/lab_usage.awk index 5deaec5..51658dd 100755 --- a/lab_usage.awk +++ b/lab_usage.awk @@ -1,5 +1,5 @@ #!/bin/awk -f -@include "mycolors.awk" +@include "/nethome/mxu87/slurm_usage_utils/mycolors.awk" function print_row_separator() { @@ -63,6 +63,11 @@ END { if (lab == "guest-lab") { continue; } + if (length(labfilter) != 0) { + if (substr(lab, 0, length(lab)-4) != substr(labfilter,3)) { + continue; + } + } print_str = sprintf(\ "[ %d/%d/%d ]", labs_to_gpus_used[lab]["R"], diff --git a/lab_usage_qos.awk b/lab_usage_qos.awk index de5b9a3..d058bc4 100644 --- a/lab_usage_qos.awk +++ b/lab_usage_qos.awk @@ -1,5 +1,5 @@ #!/bin/awk -f -@include "mycolors.awk" +@include "/nethome/mxu87/slurm_usage_utils/mycolors.awk" BEGIN { FS="[ :|=,]"; @@ -46,6 +46,11 @@ END { if (lab == "guest-lab") { continue; } + if (length(labfilter) != 0) { + if (substr(lab, 0, length(lab)-4) != substr(labfilter,3)) { + continue; + } + } print_str = sprintf("[ %d/%d/%d ]", labs_to_gpus_used[lab]["R"], labs_to_gpus_used[lab]["PD"], labs_to_gpus[lab]) printf("| %14s | %-15s |\n", lab, print_str); if (lab in gpu_counts) { diff --git a/lab_usage_verbose.awk b/lab_usage_verbose.awk index 32f481d..25895e9 100755 --- a/lab_usage_verbose.awk +++ b/lab_usage_verbose.awk @@ -1,5 +1,5 @@ #!/bin/awk -f -@include "mycolors.awk" +@include "/nethome/mxu87/slurm_usage_utils/mycolors.awk" BEGIN { FS="[ :|=,]"; @@ -90,6 +90,11 @@ END { if (lab == "guest-lab") { continue; } + if (length(labfilter) != 0) { + if (substr(lab, 0, length(lab)-4) != substr(labfilter,3)) { + continue; + } + } print_str = sprintf(\ "[ %d / %d / %d (Run/Sched/Total) GPUS]", labs_to_gpus_used[lab]["R"], diff --git a/node_usage.awk b/node_usage.awk index 159b3a7..abb49b7 100644 --- a/node_usage.awk +++ b/node_usage.awk @@ -1,5 +1,5 @@ #!/bin/awk -f -@include "mycolors.awk" +@include "/nethome/mxu87/slurm_usage_utils/mycolors.awk" BEGIN { FS="[ :|=*]"; From 7e0c278289b7cfedfdee28fc41c1710a9de16e8a Mon Sep 17 00:00:00 2001 From: Maxwell Alexander Xu Date: Thu, 5 Oct 2023 17:18:05 -0400 Subject: [PATCH 2/2] removed maxs specific absolute path --- gpu_types_usage.awk | 2 +- gpu_types_usage_verbose.awk | 2 +- lab_usage.awk | 2 +- lab_usage_qos.awk | 2 +- lab_usage_verbose.awk | 2 +- node_usage.awk | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gpu_types_usage.awk b/gpu_types_usage.awk index 549c17a..c0a7871 100644 --- a/gpu_types_usage.awk +++ b/gpu_types_usage.awk @@ -1,5 +1,5 @@ #!/bin/awk -f -@include "/nethome/mxu87/slurm_usage_utils/mycolors.awk" +@include "mycolors.awk" BEGIN { FS="[ :|=,]"; diff --git a/gpu_types_usage_verbose.awk b/gpu_types_usage_verbose.awk index f779877..19fe122 100644 --- a/gpu_types_usage_verbose.awk +++ b/gpu_types_usage_verbose.awk @@ -1,5 +1,5 @@ #!/bin/awk -f -@include "/nethome/mxu87/slurm_usage_utils/mycolors.awk" +@include "mycolors.awk" BEGIN { FS="[ :|=,]"; diff --git a/lab_usage.awk b/lab_usage.awk index 51658dd..922689d 100755 --- a/lab_usage.awk +++ b/lab_usage.awk @@ -1,5 +1,5 @@ #!/bin/awk -f -@include "/nethome/mxu87/slurm_usage_utils/mycolors.awk" +@include "mycolors.awk" function print_row_separator() { diff --git a/lab_usage_qos.awk b/lab_usage_qos.awk index d058bc4..f0444dc 100644 --- a/lab_usage_qos.awk +++ b/lab_usage_qos.awk @@ -1,5 +1,5 @@ #!/bin/awk -f -@include "/nethome/mxu87/slurm_usage_utils/mycolors.awk" +@include "mycolors.awk" BEGIN { FS="[ :|=,]"; diff --git a/lab_usage_verbose.awk b/lab_usage_verbose.awk index 25895e9..4611cdc 100755 --- a/lab_usage_verbose.awk +++ b/lab_usage_verbose.awk @@ -1,5 +1,5 @@ #!/bin/awk -f -@include "/nethome/mxu87/slurm_usage_utils/mycolors.awk" +@include "mycolors.awk" BEGIN { FS="[ :|=,]"; diff --git a/node_usage.awk b/node_usage.awk index abb49b7..159b3a7 100644 --- a/node_usage.awk +++ b/node_usage.awk @@ -1,5 +1,5 @@ #!/bin/awk -f -@include "/nethome/mxu87/slurm_usage_utils/mycolors.awk" +@include "mycolors.awk" BEGIN { FS="[ :|=*]";