Refactor splitConfigFile to use sed-based slicing (#1870)

Author: dorde-antic

mlir/utils/jenkins/Jenkinsfile

@@ -227,12 +227,12 @@ void preMergeCheckPackage(String codepath) {
 
 void splitConfigFile(String inputFilePath, String outputFilePath, int run, int totalSplits = 5) {
     sh """
-    totalLines=\$(grep -cv "^#" ${inputFilePath})
-    linesPerSplit=\$(((totalLines + ${totalSplits} - 1) / ${totalSplits}))
-    startLine=\$(((${run} - 1) * linesPerSplit + 1))
-    endLine=\$((${run} * linesPerSplit))
-
-    awk 'BEGIN{count=0} !/^#/ {count++} {if (!/^#/ && count >= '\"\${startLine}\"' && count <= '\"\${endLine}\"') print' ${inputFilePath} > ${outputFilePath}}
+    lines=\$(grep -Ev '(^\\s*\$|^\\s*#)' ${inputFilePath} | wc -l)
+    lines_per_chunk=\$(((lines + ${totalSplits} - 1) / ${totalSplits}))
+    start_line=\$((lines_per_chunk * (${run} - 1) + 1))
+    end_line=\$((lines_per_chunk * ${run}))
+    
+    grep -Ev '(^\\s*\$|^\\s*#)' ${inputFilePath} | sed -n "\${start_line},\${end_line}p" | tee ${outputFilePath}
     """
 }
 
@@ -1210,17 +1210,14 @@ pipeline {
                                         "HOME=${env.WORKSPACE}"
                                     ]) {
                                         dir('build') {
+                                            def convInput = "${WORKSPACE}/mlir/utils/performance/configs/tier1-conv-configs"
+                                            def convToUse = "${WORKSPACE}/build/tier1-conv-configs" 
+                                            def gemmInput = "${WORKSPACE}/mlir/utils/performance/configs/tier1-gemm-configs"
+                                            def gemmToUse = "${WORKSPACE}/build/tier1-gemm-configs"
                                             script {
-                                                def convInput = "${WORKSPACE}/mlir/utils/performance/configs/tier1-conv-configs"
-                                                def convToUse = convInput
                                                 if (params.nightly) {
                                                     def runIndex = ((env.BUILD_NUMBER as int) - 1) % 5 + 1
                                                     splitConfigFile(convInput, convToUse, runIndex)
-                                                }
-                                                def gemmInput = "${WORKSPACE}/mlir/utils/performance/configs/tier1-gemm-configs"
-                                                def gemmToUse = gemmInput
-                                                if (params.nightly) {
-                                                    def runIndex = ((env.BUILD_NUMBER as int) - 1) % 5 + 1
                                                     splitConfigFile(gemmInput, gemmToUse, runIndex)
                                                 }
                                             }
@@ -1290,9 +1287,9 @@ pipeline {
                                         "HOME=${env.WORKSPACE}"
                                     ]) {
                                         dir('build') {
+                                            def attnInput = "${WORKSPACE}/mlir/utils/performance/configs/tier1-attention-configs"
+                                            def attnToUse = "${WORKSPACE}/build/tier1-attention-configs"
                                             script {
-                                                def attnInput = "${WORKSPACE}/mlir/utils/performance/configs/tier1-attention-configs"
-                                                def attnToUse = attnInput
                                                 if (params.nightly) {
                                                     def runIndex = ((env.BUILD_NUMBER as int) - 1) % 5 + 1
                                                     splitConfigFile(attnInput, attnToUse, runIndex)
@@ -1352,9 +1349,9 @@ pipeline {
 
 
                                             dir('build') {
+                                                def gemmInput = "${WORKSPACE}/mlir/utils/performance/configs/tier1-gemm-configs"
+                                                def gemmToUse = "${WORKSPACE}/build/tier1-gemm-configs"
                                                 script {
-                                                    def gemmInput = "${WORKSPACE}/mlir/utils/performance/configs/tier1-gemm-configs"
-                                                    def gemmToUse = gemmInput
                                                     if (params.nightly) {
                                                         def runIndex = ((env.BUILD_NUMBER as int) - 1) % 5 + 1
                                                         splitConfigFile(gemmInput, gemmToUse, runIndex)

mlir/utils/performance/parameterSweeps.py

@@ -449,7 +449,7 @@ def main() -> bool:
         help="The build directory of MLIR based kernel generator",
     )
     args = parser.parse_args()
-    arch = ','.join(getArch())
+    arch = getArch()
     supported_codepath = ['mfma', 'vanilla', 'wmma']
     # If codepath not provided or not supported, infer it from the arch
     codepath = args.codepath

mlir/utils/performance/perfRunner.py

@@ -151,21 +151,20 @@ def getNanoSeconds(fileName):
         return np.nan
     with open(fileName, 'r') as csv_file:
         reader = csv.DictReader(csv_file, delimiter = ',')
-
         result = 0
         for row in reader:
             result += int(float(row['AverageNs']))
         csv_file.close()
         return result
 
-def getProfilerOutputPath(arch, baseOutPath):
+def getProfilerOutputPath(arch: str, baseOutPath):
     chip = GFX_CHIP_RE.search(arch).group(0)
     # TODO (gfx950): check if gfx950 need this
     if(chip not in ["gfx942"]):
         return os.path.join('pmc_1', baseOutPath)
     return baseOutPath
 
-def getMetricArgsForRocprof(arch):
+def getMetricArgsForRocprof(arch: str):
     chip = GFX_CHIP_RE.search(arch).group(0)
     current_dir = os.path.dirname(os.path.abspath(__file__))
     metrics_path = os.path.join(current_dir, ROCMLIR_INPUT_METRICS_FILE_NAME)
@@ -1750,16 +1749,19 @@ def hip_check(call_result):
         raise RuntimeError(str(err))
     return result
 
-def getArch():
+def getArch() -> str:
     agents = set()
     device_count = hip_check(hip.hipGetDeviceCount())
     for device in range(device_count):
         props = hip.hipDeviceProp_t()
         hip_check(hip.hipGetDeviceProperties(props,device))
         agent = props.gcnArchName.decode('utf-8')
         agents.add(agent)
-
-    return agents
+    if(len(agents) > 1):
+        print(f"WARNING: Found {len(agents)} different kinds of agents on the same machine :  {', '.join(agents)}")
+        print("WARNING: Using the first agent by default. If you want to use a different agent, please set the HIP_VISIBLE_DEVICES environment variable.")
+    # select first agent by default
+    return list(agents)[0]
 
 def parseDataTypes(data_types):
     if not data_types:
@@ -1779,8 +1781,7 @@ def parseDataTypes(data_types):
     return datatypes, outMap
 
 def getChip():
-    archNames = getArch()
-    arch = ','.join(archNames)
+    arch = getArch()
     chip = GFX_CHIP_RE.search(arch).group(0)
     return chip
 
@@ -1838,9 +1839,8 @@ def main(args=None):
     if args is None:
         args = sys.argv[1:]
 
-    archNames = getArch()
-    arch = ','.join(archNames)
-    chip = GFX_CHIP_RE.search(arch).group(0)
+    arch = getArch()
+    chip = getChip() 
     numCU = getNumCU(chip)
 
     root_dir = str(subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).decode().strip())

mlir/utils/performance/tuningRunner.py

@@ -240,8 +240,7 @@ def main(args=None):
     if args is None:
         args = sys.argv[1:]
 
-    archNames = perfRunner.getArch()
-    arch = ','.join(archNames)
+    arch = perfRunner.getArch()
     numCU = perfRunner.getNumCU(perfRunner.getChip())
     root_dir = str(subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).decode().strip())
     default_conv_configs = root_dir + '/mlir/utils/jenkins/performance/configs/tier1-conv-configs'