[AMDGPU][test]added unique and sort options for update_mc_test_check script (#111769)

broxigarchen · web-flow · commit 528e975ac408 · 2024-10-29T13:48:43.000-04:00
add a unique and a sort option to the update_mc_test_check script.

These mc asm/dasm files are usually large in number of lines, and these
lines are mostly similar to each other. These options can be useful when
maintainer is merging or resolving conflicts by making the file
identifical

Also fixed a small issue in asm/dasm such that the auto generated header
line is
1. asm using ";" instead of "//" as comment marker
2. dasm using ";" instead of "#" as comment marker
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm.s.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm.s.expected
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
 // RUN: llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
 
 v_bfrev_b32 v5, v1
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_err.s.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_err.s.expected
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
 // RUN: not llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
 
 v_bfrev_b32 v5, v299
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort.s b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort.s
@@ -0,0 +1,5 @@
+// RUN: llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
+
+v_bfrev_b32 v5, v1
+
+v_bfrev_b32 v1, v1
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort.s.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort.s.expected
@@ -0,0 +1,8 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort
+// RUN: llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
+
+v_bfrev_b32 v1, v1
+// CHECK: v_bfrev_b32_e32 v1, v1                  ; encoding: [0x01,0x71,0x02,0x7e]
+
+v_bfrev_b32 v5, v1
+// CHECK: v_bfrev_b32_e32 v5, v1                  ; encoding: [0x01,0x71,0x0a,0x7e]
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort_with_comment.s b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort_with_comment.s
@@ -0,0 +1,9 @@
+// RUN: llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
+
+v_bfrev_b32 v5, v1 //This is comment A
+
+v_bfrev_b32 v1, v1
+// This is comment B
+
+// This is comment C
+v_bfrev_b32 v2, v1
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort_with_comment.s.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_sort_with_comment.s.expected
@@ -0,0 +1,13 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --sort
+// RUN: llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
+
+v_bfrev_b32 v1, v1
+// CHECK: v_bfrev_b32_e32 v1, v1                  ; encoding: [0x01,0x71,0x02,0x7e]
+// This is comment B
+
+// This is comment C
+v_bfrev_b32 v2, v1
+// CHECK: v_bfrev_b32_e32 v2, v1                  ; encoding: [0x01,0x71,0x04,0x7e]
+
+v_bfrev_b32 v5, v1 //This is comment A
+// CHECK: v_bfrev_b32_e32 v5, v1                  ; encoding: [0x01,0x71,0x0a,0x7e]
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_unique.s b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_unique.s
@@ -0,0 +1,10 @@
+// RUN: llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
+
+//this is commentA
+v_bfrev_b32 v5, v1
+
+v_bfrev_b32 v5, v1
+
+//this is commentB
+
+//this is commentB
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_unique.s.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_asm_unique.s.expected
@@ -0,0 +1,10 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique
+// RUN: llvm-mc -triple=amdgcn -show-encoding %s 2>&1 | FileCheck --check-prefixes=CHECK %s
+
+//this is commentA
+v_bfrev_b32 v5, v1
+// CHECK: v_bfrev_b32_e32 v5, v1                  ; encoding: [0x01,0x71,0x0a,0x7e]
+
+//this is commentB
+
+//this is commentB
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm.txt.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm.txt.expected
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -disassemble -show-encoding %s 2>&1 | FileCheck -check-prefixes=CHECK %s
 
 0x00,0x00,0x00,0x7e
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm_unique.txt b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm_unique.txt
@@ -0,0 +1,5 @@
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -disassemble -show-encoding %s 2>&1 | FileCheck -check-prefixes=CHECK %s
+
+0x00,0x00,0x00,0x7e
+
+0x00,0x00,0x00,0x7e
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm_unique.txt.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_dasm_unique.txt.expected
@@ -0,0 +1,5 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -disassemble -show-encoding %s 2>&1 | FileCheck -check-prefixes=CHECK %s
+
+0x00,0x00,0x00,0x7e
+# CHECK: v_nop                                   ; encoding: [0x00,0x00,0x00,0x7e]
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_multirun_dasm.txt.expected b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/Inputs/amdgpu_multirun_dasm.txt.expected
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py
 # RUN: llvm-mc -triple=amdgcn -mcpu=tonga -disassemble -show-encoding %s 2>&1 | FileCheck -check-prefixes=CHECK,CHECKA %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -disassemble -show-encoding %s 2>&1 | FileCheck -check-prefixes=CHECK,CHECKB %s
 
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/amdgpu-sort.test b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/amdgpu-sort.test
@@ -0,0 +1,7 @@
+# REQUIRES: amdgpu-registered-target
+## Check that sort is working
+
+# RUN: cp -f %S/Inputs/amdgpu_asm_sort.s %t.s && %update_mc_test_checks --sort %t.s
+# RUN: diff -u %S/Inputs/amdgpu_asm_sort.s.expected %t.s
+# RUN: cp -f %S/Inputs/amdgpu_asm_sort_with_comment.s %t.s && %update_mc_test_checks --sort %t.s
+# RUN: diff -u %S/Inputs/amdgpu_asm_sort_with_comment.s.expected %t.s
diff --git a/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/amdgpu-unique.test b/llvm/test/tools/UpdateTestChecks/update_mc_test_checks/amdgpu-unique.test
@@ -0,0 +1,7 @@
+# REQUIRES: amdgpu-registered-target
+## Check that unique is working
+
+# RUN: cp -f %S/Inputs/amdgpu_asm_unique.s %t.s && %update_mc_test_checks --unique %t.s
+# RUN: diff -u %S/Inputs/amdgpu_asm_unique.s.expected %t.s
+# RUN: cp -f %S/Inputs/amdgpu_dasm_unique.txt %t.txt && %update_mc_test_checks --unique %t.txt
+# RUN: diff -u %S/Inputs/amdgpu_dasm_unique.txt.expected %t.txt
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
@@ -275,8 +275,10 @@ def __init__(
         self.run_lines = find_run_lines(test, self.input_lines)
         self.comment_prefix = comment_prefix
         if self.comment_prefix is None:
-            if self.path.endswith(".mir"):
+            if self.path.endswith(".mir") or self.path.endswith(".txt"):
                 self.comment_prefix = "#"
+            elif self.path.endswith(".s"):
+                self.comment_prefix = "//"
             else:
                 self.comment_prefix = ";"
         self.autogenerated_note_prefix = self.comment_prefix + " " + UTC_ADVERT
diff --git a/llvm/utils/update_mc_test_checks.py b/llvm/utils/update_mc_test_checks.py
@@ -6,6 +6,7 @@
 from __future__ import print_function
 
 import argparse
+import functools
 import os  # Used to advertise this file's name ("autogenerated_note").
 
 from UpdateTestChecks import common
@@ -50,6 +51,10 @@ def isTestLine(input_line, mc_mode):
     return True
 
 
+def isRunLine(l):
+    return common.RUN_LINE_RE.match(l)
+
+
 def hasErr(err):
     return err and ERROR_RE.search(err) is not None
 
@@ -118,6 +123,19 @@ def main():
         default=None,
         help="Set a default -march for when neither triple nor arch are found in a RUN line",
     )
+    parser.add_argument(
+        "--unique",
+        action="store_true",
+        default=False,
+        help="remove duplicated test line if found",
+    )
+    parser.add_argument(
+        "--sort",
+        action="store_true",
+        default=False,
+        help="sort testline in alphabetic order (keep run-lines on top), this option could be dangerous as it"
+        "could change the order of lines that are not expected",
+    )
     parser.add_argument("tests", nargs="+")
     initial_args = common.parse_commandline_args(parser)
 
@@ -130,6 +148,11 @@ def main():
             mc_mode = "asm"
         elif ti.path.endswith(".txt"):
             mc_mode = "dasm"
+
+            if ti.args.sort:
+                print("sorting with dasm(.txt) file is not supported!")
+                return -1
+
         else:
             common.warn("Expected .s and .txt, Skipping file : ", ti.path)
             continue
@@ -196,6 +219,10 @@ def main():
 
         # find all test line from input
         testlines = [l for l in ti.input_lines if isTestLine(l, mc_mode)]
+        # remove duplicated lines to save running time
+        testlines = list(dict.fromkeys(testlines))
+        common.debug("Valid test line found: ", len(testlines))
+
         run_list_size = len(run_list)
         testnum = len(testlines)
 
@@ -233,7 +260,7 @@ def main():
             raw_prefixes.append(prefixes)
 
         output_lines = []
-        generated_prefixes = []
+        generated_prefixes = {}
         used_prefixes = set()
         prefix_set = set([prefix for p in run_list for prefix in p[0]])
         common.debug("Rewriting FileCheck prefixes:", str(prefix_set))
@@ -298,23 +325,72 @@ def main():
                     else:
                         gen_prefix += getStdCheckLine(prefix, o, mc_mode)
 
-            generated_prefixes.append(gen_prefix.rstrip("\n"))
+            generated_prefixes[input_line] = gen_prefix.rstrip("\n")
 
         # write output
-        prefix_id = 0
         for input_info in ti.iterlines(output_lines):
             input_line = input_info.line
-            if isTestLine(input_line, mc_mode):
+            if input_line in testlines:
                 output_lines.append(input_line)
-                output_lines.append(generated_prefixes[prefix_id])
-                prefix_id += 1
+                output_lines.append(generated_prefixes[input_line])
 
             elif should_add_line_to_output(input_line, prefix_set, mc_mode):
                 output_lines.append(input_line)
 
-            elif input_line in ti.run_lines or input_line == "":
-                output_lines.append(input_line)
+        if ti.args.unique or ti.args.sort:
+            # split with double newlines
+            test_units = "\n".join(output_lines).split("\n\n")
+
+            # select the key line for each test unit
+            test_dic = {}
+            for unit in test_units:
+                lines = unit.split("\n")
+                for l in lines:
+                    # if contains multiple lines, use
+                    # the first testline or runline as key
+                    if isTestLine(l, mc_mode):
+                        test_dic[unit] = l
+                        break
+                    if isRunLine(l):
+                        test_dic[unit] = l
+                        break
+
+            # unique
+            if ti.args.unique:
+                new_test_units = []
+                written_lines = set()
+                for unit in test_units:
+                    # if not testline/runline, we just add it
+                    if unit not in test_dic:
+                        new_test_units.append(unit)
+                    else:
+                        if test_dic[unit] in written_lines:
+                            common.debug("Duplicated test skipped: ", unit)
+                            continue
+
+                        written_lines.add(test_dic[unit])
+                        new_test_units.append(unit)
+                test_units = new_test_units
+
+            # sort
+            if ti.args.sort:
+
+                def getkey(l):
+                    # find key of test unit, otherwise use first line
+                    if l in test_dic:
+                        line = test_dic[l]
+                    else:
+                        line = l.split("\n")[0]
+
+                    # runline placed on the top
+                    return (not isRunLine(line), line)
+
+                test_units = sorted(test_units, key=getkey)
+
+            # join back to be output string
+            output_lines = "\n\n".join(test_units).split("\n")
 
+        # output
         if ti.args.gen_unused_prefix_body:
             output_lines.extend(
                 ti.get_checks_for_unused_prefixes(run_list, used_prefixes)

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py`
	`1`	`+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py`
`2`	`2`	`// RUN: llvm-mc -triple=amdgcn -show-encoding %s 2>&1 \| FileCheck --check-prefixes=CHECK %s`
`3`	`3`
`4`	`4`	`v_bfrev_b32 v5, v1`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-; NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py`
	`1`	`+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py`
`2`	`2`	`# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -disassemble -show-encoding %s 2>&1 \| FileCheck -check-prefixes=CHECK %s`
`3`	`3`
`4`	`4`	`0x00,0x00,0x00,0x7e`